drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW)
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76
  77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  79 #define BNXT_RX_COPY_THRESH 256
  80
  81 #define BNXT_TX_PUSH_THRESH 164
  82
  83 enum board_idx {
  84         BCM57301,
  85         BCM57302,
  86         BCM57304,
  87         BCM57417_NPAR,
  88         BCM58700,
  89         BCM57311,
  90         BCM57312,
  91         BCM57402,
  92         BCM57404,
  93         BCM57406,
  94         BCM57402_NPAR,
  95         BCM57407,
  96         BCM57412,
  97         BCM57414,
  98         BCM57416,
  99         BCM57417,
 100         BCM57412_NPAR,
 101         BCM57314,
 102         BCM57417_SFP,
 103         BCM57416_SFP,
 104         BCM57404_NPAR,
 105         BCM57406_NPAR,
 106         BCM57407_SFP,
 107         BCM57407_NPAR,
 108         BCM57414_NPAR,
 109         BCM57416_NPAR,
 110         BCM57452,
 111         BCM57454,
 112         BCM5745x_NPAR,
 113         BCM57508,
 114         BCM57504,
 115         BCM57502,
 116         BCM57508_NPAR,
 117         BCM57504_NPAR,
 118         BCM57502_NPAR,
 119         BCM58802,
 120         BCM58804,
 121         BCM58808,
 122         NETXTREME_E_VF,
 123         NETXTREME_C_VF,
 124         NETXTREME_S_VF,
 125         NETXTREME_E_P5_VF,
 126 };
 127
 128 /* indexed by enum above */
 129 static const struct {
 130         char *name;
 131 } board_info[] = {
 132         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 133         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 134         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 135         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 136         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 137         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 138         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 139         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 140         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 142         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 143         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 144         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 145         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 146         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 148         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 149         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 150         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 151         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 152         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 153         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 154         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 155         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 156         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 157         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 158         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 159         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 161         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 162         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 163         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 164         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 165         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 166         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 167         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 168         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 169         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 170         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 171         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 172         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 173         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 174 };
 175
 176 static const struct pci_device_id bnxt_pci_tbl[] = {
 177         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 180         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 182         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 183         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 184         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 188         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 189         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 190         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 195         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 197         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 199         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 200         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 201         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 202         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 203         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 211         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 212         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 213         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 214         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 215         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 222         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 223 #ifdef CONFIG_BNXT_SRIOV
 224         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 225         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 226         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 227         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 234         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 235 #endif
 236         { 0 }
 237 };
 238
 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 240
 241 static const u16 bnxt_vf_req_snif[] = {
 242         HWRM_FUNC_CFG,
 243         HWRM_FUNC_VF_CFG,
 244         HWRM_PORT_PHY_QCFG,
 245         HWRM_CFA_L2_FILTER_ALLOC,
 246 };
 247
 248 static const u16 bnxt_async_events_arr[] = {
 249         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 250         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 251         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 252         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 253         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 254         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 255         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 257         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 258         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 259 };
 260
 261 static struct workqueue_struct *bnxt_pf_wq;
 262
 263 static bool bnxt_vf_pciid(enum board_idx idx)
 264 {
 265         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 266                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 267 }
 268
 269 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 270 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 271 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 272
 273 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 274                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 275
 276 #define BNXT_DB_CQ(db, idx)                                             \
 277         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 278
 279 #define BNXT_DB_NQ_P5(db, idx)                                          \
 280         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 281
 282 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 283         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 284
 285 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 286         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 287
 288 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 289 {
 290         if (bp->flags & BNXT_FLAG_CHIP_P5)
 291                 BNXT_DB_NQ_P5(db, idx);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 297 {
 298         if (bp->flags & BNXT_FLAG_CHIP_P5)
 299                 BNXT_DB_NQ_ARM_P5(db, idx);
 300         else
 301                 BNXT_DB_CQ_ARM(db, idx);
 302 }
 303
 304 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 305 {
 306         if (bp->flags & BNXT_FLAG_CHIP_P5)
 307                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 308                        db->doorbell);
 309         else
 310                 BNXT_DB_CQ(db, idx);
 311 }
 312
 313 const u16 bnxt_lhint_arr[] = {
 314         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 315         TX_BD_FLAGS_LHINT_512_TO_1023,
 316         TX_BD_FLAGS_LHINT_1024_TO_2047,
 317         TX_BD_FLAGS_LHINT_1024_TO_2047,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 332         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 333 };
 334
 335 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 336 {
 337         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 338
 339         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 340                 return 0;
 341
 342         return md_dst->u.port_info.port_id;
 343 }
 344
 345 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 346 {
 347         struct bnxt *bp = netdev_priv(dev);
 348         struct tx_bd *txbd;
 349         struct tx_bd_ext *txbd1;
 350         struct netdev_queue *txq;
 351         int i;
 352         dma_addr_t mapping;
 353         unsigned int length, pad = 0;
 354         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 355         u16 prod, last_frag;
 356         struct pci_dev *pdev = bp->pdev;
 357         struct bnxt_tx_ring_info *txr;
 358         struct bnxt_sw_tx_bd *tx_buf;
 359
 360         i = skb_get_queue_mapping(skb);
 361         if (unlikely(i >= bp->tx_nr_rings)) {
 362                 dev_kfree_skb_any(skb);
 363                 return NETDEV_TX_OK;
 364         }
 365
 366         txq = netdev_get_tx_queue(dev, i);
 367         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 368         prod = txr->tx_prod;
 369
 370         free_size = bnxt_tx_avail(bp, txr);
 371         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 372                 netif_tx_stop_queue(txq);
 373                 return NETDEV_TX_BUSY;
 374         }
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 401                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 402                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 403                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 404                 void __iomem *db = txr->tx_db.doorbell;
 405                 void *pdata = tx_push_buf->data;
 406                 u64 *end;
 407                 int j, push_len;
 408
 409                 /* Set COAL_NOW to be ready quickly for the next push */
 410                 tx_push->tx_bd_len_flags_type =
 411                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 412                                         TX_BD_TYPE_LONG_TX_BD |
 413                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 414                                         TX_BD_FLAGS_COAL_NOW |
 415                                         TX_BD_FLAGS_PACKET_END |
 416                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 417
 418                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 419                         tx_push1->tx_bd_hsize_lflags =
 420                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 421                 else
 422                         tx_push1->tx_bd_hsize_lflags = 0;
 423
 424                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 425                 tx_push1->tx_bd_cfa_action =
 426                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 427
 428                 end = pdata + length;
 429                 end = PTR_ALIGN(end, 8) - 1;
 430                 *end = 0;
 431
 432                 skb_copy_from_linear_data(skb, pdata, len);
 433                 pdata += len;
 434                 for (j = 0; j < last_frag; j++) {
 435                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 436                         void *fptr;
 437
 438                         fptr = skb_frag_address_safe(frag);
 439                         if (!fptr)
 440                                 goto normal_tx;
 441
 442                         memcpy(pdata, fptr, skb_frag_size(frag));
 443                         pdata += skb_frag_size(frag);
 444                 }
 445
 446                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 447                 txbd->tx_bd_haddr = txr->data_mapping;
 448                 prod = NEXT_TX(prod);
 449                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 450                 memcpy(txbd, tx_push1, sizeof(*txbd));
 451                 prod = NEXT_TX(prod);
 452                 tx_push->doorbell =
 453                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 454                 txr->tx_prod = prod;
 455
 456                 tx_buf->is_push = 1;
 457                 netdev_tx_sent_queue(txq, skb->len);
 458                 wmb();  /* Sync is_push and byte queue before pushing data */
 459
 460                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 461                 if (push_len > 16) {
 462                         __iowrite64_copy(db, tx_push_buf, 16);
 463                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 464                                          (push_len - 16) << 1);
 465                 } else {
 466                         __iowrite64_copy(db, tx_push_buf, push_len);
 467                 }
 468
 469                 goto tx_done;
 470         }
 471
 472 normal_tx:
 473         if (length < BNXT_MIN_PKT_SIZE) {
 474                 pad = BNXT_MIN_PKT_SIZE - length;
 475                 if (skb_pad(skb, pad)) {
 476                         /* SKB already freed. */
 477                         tx_buf->skb = NULL;
 478                         return NETDEV_TX_OK;
 479                 }
 480                 length = BNXT_MIN_PKT_SIZE;
 481         }
 482
 483         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 484
 485         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 486                 dev_kfree_skb_any(skb);
 487                 tx_buf->skb = NULL;
 488                 return NETDEV_TX_OK;
 489         }
 490
 491         dma_unmap_addr_set(tx_buf, mapping, mapping);
 492         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 493                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 494
 495         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 496
 497         prod = NEXT_TX(prod);
 498         txbd1 = (struct tx_bd_ext *)
 499                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 500
 501         txbd1->tx_bd_hsize_lflags = 0;
 502         if (skb_is_gso(skb)) {
 503                 u32 hdr_len;
 504
 505                 if (skb->encapsulation)
 506                         hdr_len = skb_inner_network_offset(skb) +
 507                                 skb_inner_network_header_len(skb) +
 508                                 inner_tcp_hdrlen(skb);
 509                 else
 510                         hdr_len = skb_transport_offset(skb) +
 511                                 tcp_hdrlen(skb);
 512
 513                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 514                                         TX_BD_FLAGS_T_IPID |
 515                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 516                 length = skb_shinfo(skb)->gso_size;
 517                 txbd1->tx_bd_mss = cpu_to_le32(length);
 518                 length += hdr_len;
 519         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 520                 txbd1->tx_bd_hsize_lflags =
 521                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 522                 txbd1->tx_bd_mss = 0;
 523         }
 524
 525         length >>= 9;
 526         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 527                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 528                                      skb->len);
 529                 i = 0;
 530                 goto tx_dma_error;
 531         }
 532         flags |= bnxt_lhint_arr[length];
 533         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 534
 535         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 536         txbd1->tx_bd_cfa_action =
 537                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 538         for (i = 0; i < last_frag; i++) {
 539                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 540
 541                 prod = NEXT_TX(prod);
 542                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 543
 544                 len = skb_frag_size(frag);
 545                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 546                                            DMA_TO_DEVICE);
 547
 548                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 549                         goto tx_dma_error;
 550
 551                 tx_buf = &txr->tx_buf_ring[prod];
 552                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 553
 554                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 555
 556                 flags = len << TX_BD_LEN_SHIFT;
 557                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 558         }
 559
 560         flags &= ~TX_BD_LEN;
 561         txbd->tx_bd_len_flags_type =
 562                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 563                             TX_BD_FLAGS_PACKET_END);
 564
 565         netdev_tx_sent_queue(txq, skb->len);
 566
 567         /* Sync BD data before updating doorbell */
 568         wmb();
 569
 570         prod = NEXT_TX(prod);
 571         txr->tx_prod = prod;
 572
 573         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 574                 bnxt_db_write(bp, &txr->tx_db, prod);
 575
 576 tx_done:
 577
 578         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 579                 if (netdev_xmit_more() && !tx_buf->is_push)
 580                         bnxt_db_write(bp, &txr->tx_db, prod);
 581
 582                 netif_tx_stop_queue(txq);
 583
 584                 /* netif_tx_stop_queue() must be done before checking
 585                  * tx index in bnxt_tx_avail() below, because in
 586                  * bnxt_tx_int(), we update tx index before checking for
 587                  * netif_tx_queue_stopped().
 588                  */
 589                 smp_mb();
 590                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 591                         netif_tx_wake_queue(txq);
 592         }
 593         return NETDEV_TX_OK;
 594
 595 tx_dma_error:
 596         last_frag = i;
 597
 598         /* start back at beginning and unmap skb */
 599         prod = txr->tx_prod;
 600         tx_buf = &txr->tx_buf_ring[prod];
 601         tx_buf->skb = NULL;
 602         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 603                          skb_headlen(skb), PCI_DMA_TODEVICE);
 604         prod = NEXT_TX(prod);
 605
 606         /* unmap remaining mapped pages */
 607         for (i = 0; i < last_frag; i++) {
 608                 prod = NEXT_TX(prod);
 609                 tx_buf = &txr->tx_buf_ring[prod];
 610                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 611                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 612                                PCI_DMA_TODEVICE);
 613         }
 614
 615         dev_kfree_skb_any(skb);
 616         return NETDEV_TX_OK;
 617 }
 618
 619 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 620 {
 621         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 622         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 623         u16 cons = txr->tx_cons;
 624         struct pci_dev *pdev = bp->pdev;
 625         int i;
 626         unsigned int tx_bytes = 0;
 627
 628         for (i = 0; i < nr_pkts; i++) {
 629                 struct bnxt_sw_tx_bd *tx_buf;
 630                 struct sk_buff *skb;
 631                 int j, last;
 632
 633                 tx_buf = &txr->tx_buf_ring[cons];
 634                 cons = NEXT_TX(cons);
 635                 skb = tx_buf->skb;
 636                 tx_buf->skb = NULL;
 637
 638                 if (tx_buf->is_push) {
 639                         tx_buf->is_push = 0;
 640                         goto next_tx_int;
 641                 }
 642
 643                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 644                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 645                 last = tx_buf->nr_frags;
 646
 647                 for (j = 0; j < last; j++) {
 648                         cons = NEXT_TX(cons);
 649                         tx_buf = &txr->tx_buf_ring[cons];
 650                         dma_unmap_page(
 651                                 &pdev->dev,
 652                                 dma_unmap_addr(tx_buf, mapping),
 653                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 654                                 PCI_DMA_TODEVICE);
 655                 }
 656
 657 next_tx_int:
 658                 cons = NEXT_TX(cons);
 659
 660                 tx_bytes += skb->len;
 661                 dev_kfree_skb_any(skb);
 662         }
 663
 664         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 665         txr->tx_cons = cons;
 666
 667         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 668          * before checking for netif_tx_queue_stopped().  Without the
 669          * memory barrier, there is a small possibility that bnxt_start_xmit()
 670          * will miss it and cause the queue to be stopped forever.
 671          */
 672         smp_mb();
 673
 674         if (unlikely(netif_tx_queue_stopped(txq)) &&
 675             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 676                 __netif_tx_lock(txq, smp_processor_id());
 677                 if (netif_tx_queue_stopped(txq) &&
 678                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 679                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 680                         netif_tx_wake_queue(txq);
 681                 __netif_tx_unlock(txq);
 682         }
 683 }
 684
 685 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 686                                          struct bnxt_rx_ring_info *rxr,
 687                                          gfp_t gfp)
 688 {
 689         struct device *dev = &bp->pdev->dev;
 690         struct page *page;
 691
 692         page = page_pool_dev_alloc_pages(rxr->page_pool);
 693         if (!page)
 694                 return NULL;
 695
 696         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 697                                       DMA_ATTR_WEAK_ORDERING);
 698         if (dma_mapping_error(dev, *mapping)) {
 699                 page_pool_recycle_direct(rxr->page_pool, page);
 700                 return NULL;
 701         }
 702         *mapping += bp->rx_dma_offset;
 703         return page;
 704 }
 705
 706 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 707                                        gfp_t gfp)
 708 {
 709         u8 *data;
 710         struct pci_dev *pdev = bp->pdev;
 711
 712         data = kmalloc(bp->rx_buf_size, gfp);
 713         if (!data)
 714                 return NULL;
 715
 716         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 717                                         bp->rx_buf_use_size, bp->rx_dir,
 718                                         DMA_ATTR_WEAK_ORDERING);
 719
 720         if (dma_mapping_error(&pdev->dev, *mapping)) {
 721                 kfree(data);
 722                 data = NULL;
 723         }
 724         return data;
 725 }
 726
 727 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 728                        u16 prod, gfp_t gfp)
 729 {
 730         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 731         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 732         dma_addr_t mapping;
 733
 734         if (BNXT_RX_PAGE_MODE(bp)) {
 735                 struct page *page =
 736                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 737
 738                 if (!page)
 739                         return -ENOMEM;
 740
 741                 rx_buf->data = page;
 742                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 743         } else {
 744                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 745
 746                 if (!data)
 747                         return -ENOMEM;
 748
 749                 rx_buf->data = data;
 750                 rx_buf->data_ptr = data + bp->rx_offset;
 751         }
 752         rx_buf->mapping = mapping;
 753
 754         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 755         return 0;
 756 }
 757
 758 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 759 {
 760         u16 prod = rxr->rx_prod;
 761         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 762         struct rx_bd *cons_bd, *prod_bd;
 763
 764         prod_rx_buf = &rxr->rx_buf_ring[prod];
 765         cons_rx_buf = &rxr->rx_buf_ring[cons];
 766
 767         prod_rx_buf->data = data;
 768         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 769
 770         prod_rx_buf->mapping = cons_rx_buf->mapping;
 771
 772         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 773         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 774
 775         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 776 }
 777
 778 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 779 {
 780         u16 next, max = rxr->rx_agg_bmap_size;
 781
 782         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 783         if (next >= max)
 784                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 785         return next;
 786 }
 787
 788 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 789                                      struct bnxt_rx_ring_info *rxr,
 790                                      u16 prod, gfp_t gfp)
 791 {
 792         struct rx_bd *rxbd =
 793                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 794         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 795         struct pci_dev *pdev = bp->pdev;
 796         struct page *page;
 797         dma_addr_t mapping;
 798         u16 sw_prod = rxr->rx_sw_agg_prod;
 799         unsigned int offset = 0;
 800
 801         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 802                 page = rxr->rx_page;
 803                 if (!page) {
 804                         page = alloc_page(gfp);
 805                         if (!page)
 806                                 return -ENOMEM;
 807                         rxr->rx_page = page;
 808                         rxr->rx_page_offset = 0;
 809                 }
 810                 offset = rxr->rx_page_offset;
 811                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 812                 if (rxr->rx_page_offset == PAGE_SIZE)
 813                         rxr->rx_page = NULL;
 814                 else
 815                         get_page(page);
 816         } else {
 817                 page = alloc_page(gfp);
 818                 if (!page)
 819                         return -ENOMEM;
 820         }
 821
 822         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 823                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 824                                      DMA_ATTR_WEAK_ORDERING);
 825         if (dma_mapping_error(&pdev->dev, mapping)) {
 826                 __free_page(page);
 827                 return -EIO;
 828         }
 829
 830         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 831                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 832
 833         __set_bit(sw_prod, rxr->rx_agg_bmap);
 834         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 835         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 836
 837         rx_agg_buf->page = page;
 838         rx_agg_buf->offset = offset;
 839         rx_agg_buf->mapping = mapping;
 840         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 841         rxbd->rx_bd_opaque = sw_prod;
 842         return 0;
 843 }
 844
 845 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 846                                        struct bnxt_cp_ring_info *cpr,
 847                                        u16 cp_cons, u16 curr)
 848 {
 849         struct rx_agg_cmp *agg;
 850
 851         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 852         agg = (struct rx_agg_cmp *)
 853                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 854         return agg;
 855 }
 856
 857 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 858                                               struct bnxt_rx_ring_info *rxr,
 859                                               u16 agg_id, u16 curr)
 860 {
 861         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 862
 863         return &tpa_info->agg_arr[curr];
 864 }
 865
 866 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 867                                    u16 start, u32 agg_bufs, bool tpa)
 868 {
 869         struct bnxt_napi *bnapi = cpr->bnapi;
 870         struct bnxt *bp = bnapi->bp;
 871         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 872         u16 prod = rxr->rx_agg_prod;
 873         u16 sw_prod = rxr->rx_sw_agg_prod;
 874         bool p5_tpa = false;
 875         u32 i;
 876
 877         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 878                 p5_tpa = true;
 879
 880         for (i = 0; i < agg_bufs; i++) {
 881                 u16 cons;
 882                 struct rx_agg_cmp *agg;
 883                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 884                 struct rx_bd *prod_bd;
 885                 struct page *page;
 886
 887                 if (p5_tpa)
 888                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 889                 else
 890                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 891                 cons = agg->rx_agg_cmp_opaque;
 892                 __clear_bit(cons, rxr->rx_agg_bmap);
 893
 894                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 895                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 896
 897                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 898                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 899                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 900
 901                 /* It is possible for sw_prod to be equal to cons, so
 902                  * set cons_rx_buf->page to NULL first.
 903                  */
 904                 page = cons_rx_buf->page;
 905                 cons_rx_buf->page = NULL;
 906                 prod_rx_buf->page = page;
 907                 prod_rx_buf->offset = cons_rx_buf->offset;
 908
 909                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 910
 911                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 912
 913                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 914                 prod_bd->rx_bd_opaque = sw_prod;
 915
 916                 prod = NEXT_RX_AGG(prod);
 917                 sw_prod = NEXT_RX_AGG(sw_prod);
 918         }
 919         rxr->rx_agg_prod = prod;
 920         rxr->rx_sw_agg_prod = sw_prod;
 921 }
 922
 923 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 924                                         struct bnxt_rx_ring_info *rxr,
 925                                         u16 cons, void *data, u8 *data_ptr,
 926                                         dma_addr_t dma_addr,
 927                                         unsigned int offset_and_len)
 928 {
 929         unsigned int payload = offset_and_len >> 16;
 930         unsigned int len = offset_and_len & 0xffff;
 931         skb_frag_t *frag;
 932         struct page *page = data;
 933         u16 prod = rxr->rx_prod;
 934         struct sk_buff *skb;
 935         int off, err;
 936
 937         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 938         if (unlikely(err)) {
 939                 bnxt_reuse_rx_data(rxr, cons, data);
 940                 return NULL;
 941         }
 942         dma_addr -= bp->rx_dma_offset;
 943         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 944                              DMA_ATTR_WEAK_ORDERING);
 945         page_pool_release_page(rxr->page_pool, page);
 946
 947         if (unlikely(!payload))
 948                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 949
 950         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 951         if (!skb) {
 952                 __free_page(page);
 953                 return NULL;
 954         }
 955
 956         off = (void *)data_ptr - page_address(page);
 957         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 958         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 959                payload + NET_IP_ALIGN);
 960
 961         frag = &skb_shinfo(skb)->frags[0];
 962         skb_frag_size_sub(frag, payload);
 963         skb_frag_off_add(frag, payload);
 964         skb->data_len -= payload;
 965         skb->tail += payload;
 966
 967         return skb;
 968 }
 969
 970 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 971                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 972                                    void *data, u8 *data_ptr,
 973                                    dma_addr_t dma_addr,
 974                                    unsigned int offset_and_len)
 975 {
 976         u16 prod = rxr->rx_prod;
 977         struct sk_buff *skb;
 978         int err;
 979
 980         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 981         if (unlikely(err)) {
 982                 bnxt_reuse_rx_data(rxr, cons, data);
 983                 return NULL;
 984         }
 985
 986         skb = build_skb(data, 0);
 987         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 988                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 989         if (!skb) {
 990                 kfree(data);
 991                 return NULL;
 992         }
 993
 994         skb_reserve(skb, bp->rx_offset);
 995         skb_put(skb, offset_and_len & 0xffff);
 996         return skb;
 997 }
 998
 999 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1000                                      struct bnxt_cp_ring_info *cpr,
1001                                      struct sk_buff *skb, u16 idx,
1002                                      u32 agg_bufs, bool tpa)
1003 {
1004         struct bnxt_napi *bnapi = cpr->bnapi;
1005         struct pci_dev *pdev = bp->pdev;
1006         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1007         u16 prod = rxr->rx_agg_prod;
1008         bool p5_tpa = false;
1009         u32 i;
1010
1011         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1012                 p5_tpa = true;
1013
1014         for (i = 0; i < agg_bufs; i++) {
1015                 u16 cons, frag_len;
1016                 struct rx_agg_cmp *agg;
1017                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1018                 struct page *page;
1019                 dma_addr_t mapping;
1020
1021                 if (p5_tpa)
1022                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1023                 else
1024                         agg = bnxt_get_agg(bp, cpr, idx, i);
1025                 cons = agg->rx_agg_cmp_opaque;
1026                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1027                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1028
1029                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1030                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1031                                    cons_rx_buf->offset, frag_len);
1032                 __clear_bit(cons, rxr->rx_agg_bmap);
1033
1034                 /* It is possible for bnxt_alloc_rx_page() to allocate
1035                  * a sw_prod index that equals the cons index, so we
1036                  * need to clear the cons entry now.
1037                  */
1038                 mapping = cons_rx_buf->mapping;
1039                 page = cons_rx_buf->page;
1040                 cons_rx_buf->page = NULL;
1041
1042                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1043                         struct skb_shared_info *shinfo;
1044                         unsigned int nr_frags;
1045
1046                         shinfo = skb_shinfo(skb);
1047                         nr_frags = --shinfo->nr_frags;
1048                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1049
1050                         dev_kfree_skb(skb);
1051
1052                         cons_rx_buf->page = page;
1053
1054                         /* Update prod since possibly some pages have been
1055                          * allocated already.
1056                          */
1057                         rxr->rx_agg_prod = prod;
1058                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1059                         return NULL;
1060                 }
1061
1062                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1063                                      PCI_DMA_FROMDEVICE,
1064                                      DMA_ATTR_WEAK_ORDERING);
1065
1066                 skb->data_len += frag_len;
1067                 skb->len += frag_len;
1068                 skb->truesize += PAGE_SIZE;
1069
1070                 prod = NEXT_RX_AGG(prod);
1071         }
1072         rxr->rx_agg_prod = prod;
1073         return skb;
1074 }
1075
1076 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1077                                u8 agg_bufs, u32 *raw_cons)
1078 {
1079         u16 last;
1080         struct rx_agg_cmp *agg;
1081
1082         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1083         last = RING_CMP(*raw_cons);
1084         agg = (struct rx_agg_cmp *)
1085                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1086         return RX_AGG_CMP_VALID(agg, *raw_cons);
1087 }
1088
1089 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1090                                             unsigned int len,
1091                                             dma_addr_t mapping)
1092 {
1093         struct bnxt *bp = bnapi->bp;
1094         struct pci_dev *pdev = bp->pdev;
1095         struct sk_buff *skb;
1096
1097         skb = napi_alloc_skb(&bnapi->napi, len);
1098         if (!skb)
1099                 return NULL;
1100
1101         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1102                                 bp->rx_dir);
1103
1104         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1105                len + NET_IP_ALIGN);
1106
1107         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1108                                    bp->rx_dir);
1109
1110         skb_put(skb, len);
1111         return skb;
1112 }
1113
1114 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1115                            u32 *raw_cons, void *cmp)
1116 {
1117         struct rx_cmp *rxcmp = cmp;
1118         u32 tmp_raw_cons = *raw_cons;
1119         u8 cmp_type, agg_bufs = 0;
1120
1121         cmp_type = RX_CMP_TYPE(rxcmp);
1122
1123         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1124                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1125                             RX_CMP_AGG_BUFS) >>
1126                            RX_CMP_AGG_BUFS_SHIFT;
1127         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1128                 struct rx_tpa_end_cmp *tpa_end = cmp;
1129
1130                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1131                         return 0;
1132
1133                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1134         }
1135
1136         if (agg_bufs) {
1137                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1138                         return -EBUSY;
1139         }
1140         *raw_cons = tmp_raw_cons;
1141         return 0;
1142 }
1143
1144 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1145 {
1146         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1147                 return;
1148
1149         if (BNXT_PF(bp))
1150                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1151         else
1152                 schedule_delayed_work(&bp->fw_reset_task, delay);
1153 }
1154
1155 static void bnxt_queue_sp_work(struct bnxt *bp)
1156 {
1157         if (BNXT_PF(bp))
1158                 queue_work(bnxt_pf_wq, &bp->sp_task);
1159         else
1160                 schedule_work(&bp->sp_task);
1161 }
1162
1163 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1164 {
1165         if (!rxr->bnapi->in_reset) {
1166                 rxr->bnapi->in_reset = true;
1167                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1168                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1169                 else
1170                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1171                 bnxt_queue_sp_work(bp);
1172         }
1173         rxr->rx_next_cons = 0xffff;
1174 }
1175
1176 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1177 {
1178         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1179         u16 idx = agg_id & MAX_TPA_P5_MASK;
1180
1181         if (test_bit(idx, map->agg_idx_bmap))
1182                 idx = find_first_zero_bit(map->agg_idx_bmap,
1183                                           BNXT_AGG_IDX_BMAP_SIZE);
1184         __set_bit(idx, map->agg_idx_bmap);
1185         map->agg_id_tbl[agg_id] = idx;
1186         return idx;
1187 }
1188
1189 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1190 {
1191         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1192
1193         __clear_bit(idx, map->agg_idx_bmap);
1194 }
1195
1196 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1197 {
1198         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1199
1200         return map->agg_id_tbl[agg_id];
1201 }
1202
1203 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1204                            struct rx_tpa_start_cmp *tpa_start,
1205                            struct rx_tpa_start_cmp_ext *tpa_start1)
1206 {
1207         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1208         struct bnxt_tpa_info *tpa_info;
1209         u16 cons, prod, agg_id;
1210         struct rx_bd *prod_bd;
1211         dma_addr_t mapping;
1212
1213         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1214                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1215                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1216         } else {
1217                 agg_id = TPA_START_AGG_ID(tpa_start);
1218         }
1219         cons = tpa_start->rx_tpa_start_cmp_opaque;
1220         prod = rxr->rx_prod;
1221         cons_rx_buf = &rxr->rx_buf_ring[cons];
1222         prod_rx_buf = &rxr->rx_buf_ring[prod];
1223         tpa_info = &rxr->rx_tpa[agg_id];
1224
1225         if (unlikely(cons != rxr->rx_next_cons ||
1226                      TPA_START_ERROR(tpa_start))) {
1227                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1228                             cons, rxr->rx_next_cons,
1229                             TPA_START_ERROR_CODE(tpa_start1));
1230                 bnxt_sched_reset(bp, rxr);
1231                 return;
1232         }
1233         /* Store cfa_code in tpa_info to use in tpa_end
1234          * completion processing.
1235          */
1236         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1237         prod_rx_buf->data = tpa_info->data;
1238         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1239
1240         mapping = tpa_info->mapping;
1241         prod_rx_buf->mapping = mapping;
1242
1243         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1244
1245         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1246
1247         tpa_info->data = cons_rx_buf->data;
1248         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1249         cons_rx_buf->data = NULL;
1250         tpa_info->mapping = cons_rx_buf->mapping;
1251
1252         tpa_info->len =
1253                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1254                                 RX_TPA_START_CMP_LEN_SHIFT;
1255         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1256                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1257
1258                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1259                 tpa_info->gso_type = SKB_GSO_TCPV4;
1260                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1261                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1262                         tpa_info->gso_type = SKB_GSO_TCPV6;
1263                 tpa_info->rss_hash =
1264                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1265         } else {
1266                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1267                 tpa_info->gso_type = 0;
1268                 if (netif_msg_rx_err(bp))
1269                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1270         }
1271         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1272         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1273         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1274         tpa_info->agg_count = 0;
1275
1276         rxr->rx_prod = NEXT_RX(prod);
1277         cons = NEXT_RX(cons);
1278         rxr->rx_next_cons = NEXT_RX(cons);
1279         cons_rx_buf = &rxr->rx_buf_ring[cons];
1280
1281         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1282         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1283         cons_rx_buf->data = NULL;
1284 }
1285
1286 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1287 {
1288         if (agg_bufs)
1289                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1290 }
1291
1292 #ifdef CONFIG_INET
1293 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1294 {
1295         struct udphdr *uh = NULL;
1296
1297         if (ip_proto == htons(ETH_P_IP)) {
1298                 struct iphdr *iph = (struct iphdr *)skb->data;
1299
1300                 if (iph->protocol == IPPROTO_UDP)
1301                         uh = (struct udphdr *)(iph + 1);
1302         } else {
1303                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1304
1305                 if (iph->nexthdr == IPPROTO_UDP)
1306                         uh = (struct udphdr *)(iph + 1);
1307         }
1308         if (uh) {
1309                 if (uh->check)
1310                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1311                 else
1312                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1313         }
1314 }
1315 #endif
1316
1317 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1318                                            int payload_off, int tcp_ts,
1319                                            struct sk_buff *skb)
1320 {
1321 #ifdef CONFIG_INET
1322         struct tcphdr *th;
1323         int len, nw_off;
1324         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1325         u32 hdr_info = tpa_info->hdr_info;
1326         bool loopback = false;
1327
1328         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1329         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1330         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1331
1332         /* If the packet is an internal loopback packet, the offsets will
1333          * have an extra 4 bytes.
1334          */
1335         if (inner_mac_off == 4) {
1336                 loopback = true;
1337         } else if (inner_mac_off > 4) {
1338                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1339                                             ETH_HLEN - 2));
1340
1341                 /* We only support inner iPv4/ipv6.  If we don't see the
1342                  * correct protocol ID, it must be a loopback packet where
1343                  * the offsets are off by 4.
1344                  */
1345                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1346                         loopback = true;
1347         }
1348         if (loopback) {
1349                 /* internal loopback packet, subtract all offsets by 4 */
1350                 inner_ip_off -= 4;
1351                 inner_mac_off -= 4;
1352                 outer_ip_off -= 4;
1353         }
1354
1355         nw_off = inner_ip_off - ETH_HLEN;
1356         skb_set_network_header(skb, nw_off);
1357         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1358                 struct ipv6hdr *iph = ipv6_hdr(skb);
1359
1360                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1361                 len = skb->len - skb_transport_offset(skb);
1362                 th = tcp_hdr(skb);
1363                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1364         } else {
1365                 struct iphdr *iph = ip_hdr(skb);
1366
1367                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1368                 len = skb->len - skb_transport_offset(skb);
1369                 th = tcp_hdr(skb);
1370                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1371         }
1372
1373         if (inner_mac_off) { /* tunnel */
1374                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1375                                             ETH_HLEN - 2));
1376
1377                 bnxt_gro_tunnel(skb, proto);
1378         }
1379 #endif
1380         return skb;
1381 }
1382
1383 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1384                                            int payload_off, int tcp_ts,
1385                                            struct sk_buff *skb)
1386 {
1387 #ifdef CONFIG_INET
1388         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1389         u32 hdr_info = tpa_info->hdr_info;
1390         int iphdr_len, nw_off;
1391
1392         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1393         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1394         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1395
1396         nw_off = inner_ip_off - ETH_HLEN;
1397         skb_set_network_header(skb, nw_off);
1398         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1399                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1400         skb_set_transport_header(skb, nw_off + iphdr_len);
1401
1402         if (inner_mac_off) { /* tunnel */
1403                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1404                                             ETH_HLEN - 2));
1405
1406                 bnxt_gro_tunnel(skb, proto);
1407         }
1408 #endif
1409         return skb;
1410 }
1411
1412 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1413 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1414
1415 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1416                                            int payload_off, int tcp_ts,
1417                                            struct sk_buff *skb)
1418 {
1419 #ifdef CONFIG_INET
1420         struct tcphdr *th;
1421         int len, nw_off, tcp_opt_len = 0;
1422
1423         if (tcp_ts)
1424                 tcp_opt_len = 12;
1425
1426         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1427                 struct iphdr *iph;
1428
1429                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1430                          ETH_HLEN;
1431                 skb_set_network_header(skb, nw_off);
1432                 iph = ip_hdr(skb);
1433                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1434                 len = skb->len - skb_transport_offset(skb);
1435                 th = tcp_hdr(skb);
1436                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1437         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1438                 struct ipv6hdr *iph;
1439
1440                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1441                          ETH_HLEN;
1442                 skb_set_network_header(skb, nw_off);
1443                 iph = ipv6_hdr(skb);
1444                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1445                 len = skb->len - skb_transport_offset(skb);
1446                 th = tcp_hdr(skb);
1447                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1448         } else {
1449                 dev_kfree_skb_any(skb);
1450                 return NULL;
1451         }
1452
1453         if (nw_off) /* tunnel */
1454                 bnxt_gro_tunnel(skb, skb->protocol);
1455 #endif
1456         return skb;
1457 }
1458
1459 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1460                                            struct bnxt_tpa_info *tpa_info,
1461                                            struct rx_tpa_end_cmp *tpa_end,
1462                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1463                                            struct sk_buff *skb)
1464 {
1465 #ifdef CONFIG_INET
1466         int payload_off;
1467         u16 segs;
1468
1469         segs = TPA_END_TPA_SEGS(tpa_end);
1470         if (segs == 1)
1471                 return skb;
1472
1473         NAPI_GRO_CB(skb)->count = segs;
1474         skb_shinfo(skb)->gso_size =
1475                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1476         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1477         if (bp->flags & BNXT_FLAG_CHIP_P5)
1478                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1479         else
1480                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1481         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1482         if (likely(skb))
1483                 tcp_gro_complete(skb);
1484 #endif
1485         return skb;
1486 }
1487
1488 /* Given the cfa_code of a received packet determine which
1489  * netdev (vf-rep or PF) the packet is destined to.
1490  */
1491 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1492 {
1493         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1494
1495         /* if vf-rep dev is NULL, the must belongs to the PF */
1496         return dev ? dev : bp->dev;
1497 }
1498
1499 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1500                                            struct bnxt_cp_ring_info *cpr,
1501                                            u32 *raw_cons,
1502                                            struct rx_tpa_end_cmp *tpa_end,
1503                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1504                                            u8 *event)
1505 {
1506         struct bnxt_napi *bnapi = cpr->bnapi;
1507         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1508         u8 *data_ptr, agg_bufs;
1509         unsigned int len;
1510         struct bnxt_tpa_info *tpa_info;
1511         dma_addr_t mapping;
1512         struct sk_buff *skb;
1513         u16 idx = 0, agg_id;
1514         void *data;
1515         bool gro;
1516
1517         if (unlikely(bnapi->in_reset)) {
1518                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1519
1520                 if (rc < 0)
1521                         return ERR_PTR(-EBUSY);
1522                 return NULL;
1523         }
1524
1525         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1526                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1527                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1528                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1529                 tpa_info = &rxr->rx_tpa[agg_id];
1530                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1531                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1532                                     agg_bufs, tpa_info->agg_count);
1533                         agg_bufs = tpa_info->agg_count;
1534                 }
1535                 tpa_info->agg_count = 0;
1536                 *event |= BNXT_AGG_EVENT;
1537                 bnxt_free_agg_idx(rxr, agg_id);
1538                 idx = agg_id;
1539                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1540         } else {
1541                 agg_id = TPA_END_AGG_ID(tpa_end);
1542                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1543                 tpa_info = &rxr->rx_tpa[agg_id];
1544                 idx = RING_CMP(*raw_cons);
1545                 if (agg_bufs) {
1546                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1547                                 return ERR_PTR(-EBUSY);
1548
1549                         *event |= BNXT_AGG_EVENT;
1550                         idx = NEXT_CMP(idx);
1551                 }
1552                 gro = !!TPA_END_GRO(tpa_end);
1553         }
1554         data = tpa_info->data;
1555         data_ptr = tpa_info->data_ptr;
1556         prefetch(data_ptr);
1557         len = tpa_info->len;
1558         mapping = tpa_info->mapping;
1559
1560         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1561                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1562                 if (agg_bufs > MAX_SKB_FRAGS)
1563                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1564                                     agg_bufs, (int)MAX_SKB_FRAGS);
1565                 return NULL;
1566         }
1567
1568         if (len <= bp->rx_copy_thresh) {
1569                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1570                 if (!skb) {
1571                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1572                         return NULL;
1573                 }
1574         } else {
1575                 u8 *new_data;
1576                 dma_addr_t new_mapping;
1577
1578                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1579                 if (!new_data) {
1580                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1581                         return NULL;
1582                 }
1583
1584                 tpa_info->data = new_data;
1585                 tpa_info->data_ptr = new_data + bp->rx_offset;
1586                 tpa_info->mapping = new_mapping;
1587
1588                 skb = build_skb(data, 0);
1589                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1590                                        bp->rx_buf_use_size, bp->rx_dir,
1591                                        DMA_ATTR_WEAK_ORDERING);
1592
1593                 if (!skb) {
1594                         kfree(data);
1595                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1596                         return NULL;
1597                 }
1598                 skb_reserve(skb, bp->rx_offset);
1599                 skb_put(skb, len);
1600         }
1601
1602         if (agg_bufs) {
1603                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1604                 if (!skb) {
1605                         /* Page reuse already handled by bnxt_rx_pages(). */
1606                         return NULL;
1607                 }
1608         }
1609
1610         skb->protocol =
1611                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1612
1613         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1614                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1615
1616         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1617             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1618                 u16 vlan_proto = tpa_info->metadata >>
1619                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1620                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1621
1622                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1623         }
1624
1625         skb_checksum_none_assert(skb);
1626         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1627                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1628                 skb->csum_level =
1629                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1630         }
1631
1632         if (gro)
1633                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1634
1635         return skb;
1636 }
1637
1638 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1639                          struct rx_agg_cmp *rx_agg)
1640 {
1641         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1642         struct bnxt_tpa_info *tpa_info;
1643
1644         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1645         tpa_info = &rxr->rx_tpa[agg_id];
1646         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1647         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1648 }
1649
1650 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1651                              struct sk_buff *skb)
1652 {
1653         if (skb->dev != bp->dev) {
1654                 /* this packet belongs to a vf-rep */
1655                 bnxt_vf_rep_rx(bp, skb);
1656                 return;
1657         }
1658         skb_record_rx_queue(skb, bnapi->index);
1659         napi_gro_receive(&bnapi->napi, skb);
1660 }
1661
1662 /* returns the following:
1663  * 1       - 1 packet successfully received
1664  * 0       - successful TPA_START, packet not completed yet
1665  * -EBUSY  - completion ring does not have all the agg buffers yet
1666  * -ENOMEM - packet aborted due to out of memory
1667  * -EIO    - packet aborted due to hw error indicated in BD
1668  */
1669 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1670                        u32 *raw_cons, u8 *event)
1671 {
1672         struct bnxt_napi *bnapi = cpr->bnapi;
1673         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1674         struct net_device *dev = bp->dev;
1675         struct rx_cmp *rxcmp;
1676         struct rx_cmp_ext *rxcmp1;
1677         u32 tmp_raw_cons = *raw_cons;
1678         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1679         struct bnxt_sw_rx_bd *rx_buf;
1680         unsigned int len;
1681         u8 *data_ptr, agg_bufs, cmp_type;
1682         dma_addr_t dma_addr;
1683         struct sk_buff *skb;
1684         void *data;
1685         int rc = 0;
1686         u32 misc;
1687
1688         rxcmp = (struct rx_cmp *)
1689                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1690
1691         cmp_type = RX_CMP_TYPE(rxcmp);
1692
1693         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1694                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1695                 goto next_rx_no_prod_no_len;
1696         }
1697
1698         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1699         cp_cons = RING_CMP(tmp_raw_cons);
1700         rxcmp1 = (struct rx_cmp_ext *)
1701                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1702
1703         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1704                 return -EBUSY;
1705
1706         prod = rxr->rx_prod;
1707
1708         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1709                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1710                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1711
1712                 *event |= BNXT_RX_EVENT;
1713                 goto next_rx_no_prod_no_len;
1714
1715         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1716                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1717                                    (struct rx_tpa_end_cmp *)rxcmp,
1718                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1719
1720                 if (IS_ERR(skb))
1721                         return -EBUSY;
1722
1723                 rc = -ENOMEM;
1724                 if (likely(skb)) {
1725                         bnxt_deliver_skb(bp, bnapi, skb);
1726                         rc = 1;
1727                 }
1728                 *event |= BNXT_RX_EVENT;
1729                 goto next_rx_no_prod_no_len;
1730         }
1731
1732         cons = rxcmp->rx_cmp_opaque;
1733         if (unlikely(cons != rxr->rx_next_cons)) {
1734                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1735
1736                 /* 0xffff is forced error, don't print it */
1737                 if (rxr->rx_next_cons != 0xffff)
1738                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1739                                     cons, rxr->rx_next_cons);
1740                 bnxt_sched_reset(bp, rxr);
1741                 return rc1;
1742         }
1743         rx_buf = &rxr->rx_buf_ring[cons];
1744         data = rx_buf->data;
1745         data_ptr = rx_buf->data_ptr;
1746         prefetch(data_ptr);
1747
1748         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1749         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1750
1751         if (agg_bufs) {
1752                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1753                         return -EBUSY;
1754
1755                 cp_cons = NEXT_CMP(cp_cons);
1756                 *event |= BNXT_AGG_EVENT;
1757         }
1758         *event |= BNXT_RX_EVENT;
1759
1760         rx_buf->data = NULL;
1761         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1762                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1763
1764                 bnxt_reuse_rx_data(rxr, cons, data);
1765                 if (agg_bufs)
1766                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1767                                                false);
1768
1769                 rc = -EIO;
1770                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1771                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1772                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1773                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1774                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1775                                                  rx_err);
1776                                 bnxt_sched_reset(bp, rxr);
1777                         }
1778                 }
1779                 goto next_rx_no_len;
1780         }
1781
1782         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1783         dma_addr = rx_buf->mapping;
1784
1785         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1786                 rc = 1;
1787                 goto next_rx;
1788         }
1789
1790         if (len <= bp->rx_copy_thresh) {
1791                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1792                 bnxt_reuse_rx_data(rxr, cons, data);
1793                 if (!skb) {
1794                         if (agg_bufs)
1795                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1796                                                        agg_bufs, false);
1797                         rc = -ENOMEM;
1798                         goto next_rx;
1799                 }
1800         } else {
1801                 u32 payload;
1802
1803                 if (rx_buf->data_ptr == data_ptr)
1804                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1805                 else
1806                         payload = 0;
1807                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1808                                       payload | len);
1809                 if (!skb) {
1810                         rc = -ENOMEM;
1811                         goto next_rx;
1812                 }
1813         }
1814
1815         if (agg_bufs) {
1816                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1817                 if (!skb) {
1818                         rc = -ENOMEM;
1819                         goto next_rx;
1820                 }
1821         }
1822
1823         if (RX_CMP_HASH_VALID(rxcmp)) {
1824                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1825                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1826
1827                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1828                 if (hash_type != 1 && hash_type != 3)
1829                         type = PKT_HASH_TYPE_L3;
1830                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1831         }
1832
1833         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1834         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1835
1836         if ((rxcmp1->rx_cmp_flags2 &
1837              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1838             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1839                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1840                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1841                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1842
1843                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1844         }
1845
1846         skb_checksum_none_assert(skb);
1847         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1848                 if (dev->features & NETIF_F_RXCSUM) {
1849                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1850                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1851                 }
1852         } else {
1853                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1854                         if (dev->features & NETIF_F_RXCSUM)
1855                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1856                 }
1857         }
1858
1859         bnxt_deliver_skb(bp, bnapi, skb);
1860         rc = 1;
1861
1862 next_rx:
1863         cpr->rx_packets += 1;
1864         cpr->rx_bytes += len;
1865
1866 next_rx_no_len:
1867         rxr->rx_prod = NEXT_RX(prod);
1868         rxr->rx_next_cons = NEXT_RX(cons);
1869
1870 next_rx_no_prod_no_len:
1871         *raw_cons = tmp_raw_cons;
1872
1873         return rc;
1874 }
1875
1876 /* In netpoll mode, if we are using a combined completion ring, we need to
1877  * discard the rx packets and recycle the buffers.
1878  */
1879 static int bnxt_force_rx_discard(struct bnxt *bp,
1880                                  struct bnxt_cp_ring_info *cpr,
1881                                  u32 *raw_cons, u8 *event)
1882 {
1883         u32 tmp_raw_cons = *raw_cons;
1884         struct rx_cmp_ext *rxcmp1;
1885         struct rx_cmp *rxcmp;
1886         u16 cp_cons;
1887         u8 cmp_type;
1888
1889         cp_cons = RING_CMP(tmp_raw_cons);
1890         rxcmp = (struct rx_cmp *)
1891                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1892
1893         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1894         cp_cons = RING_CMP(tmp_raw_cons);
1895         rxcmp1 = (struct rx_cmp_ext *)
1896                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1897
1898         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1899                 return -EBUSY;
1900
1901         cmp_type = RX_CMP_TYPE(rxcmp);
1902         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1903                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1904                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1905         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1906                 struct rx_tpa_end_cmp_ext *tpa_end1;
1907
1908                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1909                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1910                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1911         }
1912         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1913 }
1914
1915 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1916 {
1917         struct bnxt_fw_health *fw_health = bp->fw_health;
1918         u32 reg = fw_health->regs[reg_idx];
1919         u32 reg_type, reg_off, val = 0;
1920
1921         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1922         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1923         switch (reg_type) {
1924         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1925                 pci_read_config_dword(bp->pdev, reg_off, &val);
1926                 break;
1927         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1928                 reg_off = fw_health->mapped_regs[reg_idx];
1929                 fallthrough;
1930         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1931                 val = readl(bp->bar0 + reg_off);
1932                 break;
1933         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1934                 val = readl(bp->bar1 + reg_off);
1935                 break;
1936         }
1937         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1938                 val &= fw_health->fw_reset_inprog_reg_mask;
1939         return val;
1940 }
1941
1942 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1943 {
1944         int i;
1945
1946         for (i = 0; i < bp->rx_nr_rings; i++) {
1947                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
1948                 struct bnxt_ring_grp_info *grp_info;
1949
1950                 grp_info = &bp->grp_info[grp_idx];
1951                 if (grp_info->agg_fw_ring_id == ring_id)
1952                         return grp_idx;
1953         }
1954         return INVALID_HW_RING_ID;
1955 }
1956
1957 #define BNXT_GET_EVENT_PORT(data)       \
1958         ((data) &                       \
1959          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1960
1961 #define BNXT_EVENT_RING_TYPE(data2)     \
1962         ((data2) &                      \
1963          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1964
1965 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
1966         (BNXT_EVENT_RING_TYPE(data2) == \
1967          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1968
1969 static int bnxt_async_event_process(struct bnxt *bp,
1970                                     struct hwrm_async_event_cmpl *cmpl)
1971 {
1972         u16 event_id = le16_to_cpu(cmpl->event_id);
1973         u32 data1 = le32_to_cpu(cmpl->event_data1);
1974         u32 data2 = le32_to_cpu(cmpl->event_data2);
1975
1976         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1977         switch (event_id) {
1978         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1979                 struct bnxt_link_info *link_info = &bp->link_info;
1980
1981                 if (BNXT_VF(bp))
1982                         goto async_event_process_exit;
1983
1984                 /* print unsupported speed warning in forced speed mode only */
1985                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1986                     (data1 & 0x20000)) {
1987                         u16 fw_speed = link_info->force_link_speed;
1988                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1989
1990                         if (speed != SPEED_UNKNOWN)
1991                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1992                                             speed);
1993                 }
1994                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1995         }
1996                 fallthrough;
1997         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1998         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1999                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2000                 fallthrough;
2001         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2002                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2003                 break;
2004         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2005                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2006                 break;
2007         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2008                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2009
2010                 if (BNXT_VF(bp))
2011                         break;
2012
2013                 if (bp->pf.port_id != port_id)
2014                         break;
2015
2016                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2017                 break;
2018         }
2019         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2020                 if (BNXT_PF(bp))
2021                         goto async_event_process_exit;
2022                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2023                 break;
2024         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
2025                 if (netif_msg_hw(bp))
2026                         netdev_warn(bp->dev, "Received RESET_NOTIFY event, data1: 0x%x, data2: 0x%x\n",
2027                                     data1, data2);
2028                 if (!bp->fw_health)
2029                         goto async_event_process_exit;
2030
2031                 bp->fw_reset_timestamp = jiffies;
2032                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2033                 if (!bp->fw_reset_min_dsecs)
2034                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2035                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2036                 if (!bp->fw_reset_max_dsecs)
2037                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2038                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2039                         netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2040                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2041                 } else {
2042                         netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2043                                     bp->fw_reset_max_dsecs * 100);
2044                 }
2045                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2046                 break;
2047         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2048                 struct bnxt_fw_health *fw_health = bp->fw_health;
2049
2050                 if (!fw_health)
2051                         goto async_event_process_exit;
2052
2053                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2054                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2055                 if (!fw_health->enabled)
2056                         break;
2057
2058                 if (netif_msg_drv(bp))
2059                         netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2060                                     fw_health->enabled, fw_health->master,
2061                                     bnxt_fw_health_readl(bp,
2062                                                          BNXT_FW_RESET_CNT_REG),
2063                                     bnxt_fw_health_readl(bp,
2064                                                          BNXT_FW_HEALTH_REG));
2065                 fw_health->tmr_multiplier =
2066                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2067                                      bp->current_interval * 10);
2068                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2069                 fw_health->last_fw_heartbeat =
2070                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2071                 fw_health->last_fw_reset_cnt =
2072                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2073                 goto async_event_process_exit;
2074         }
2075         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2076                 struct bnxt_rx_ring_info *rxr;
2077                 u16 grp_idx;
2078
2079                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2080                         goto async_event_process_exit;
2081
2082                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2083                             BNXT_EVENT_RING_TYPE(data2), data1);
2084                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2085                         goto async_event_process_exit;
2086
2087                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2088                 if (grp_idx == INVALID_HW_RING_ID) {
2089                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2090                                     data1);
2091                         goto async_event_process_exit;
2092                 }
2093                 rxr = bp->bnapi[grp_idx]->rx_ring;
2094                 bnxt_sched_reset(bp, rxr);
2095                 goto async_event_process_exit;
2096         }
2097         default:
2098                 goto async_event_process_exit;
2099         }
2100         bnxt_queue_sp_work(bp);
2101 async_event_process_exit:
2102         bnxt_ulp_async_events(bp, cmpl);
2103         return 0;
2104 }
2105
2106 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2107 {
2108         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2109         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2110         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2111                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2112
2113         switch (cmpl_type) {
2114         case CMPL_BASE_TYPE_HWRM_DONE:
2115                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2116                 if (seq_id == bp->hwrm_intr_seq_id)
2117                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2118                 else
2119                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2120                 break;
2121
2122         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2123                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2124
2125                 if ((vf_id < bp->pf.first_vf_id) ||
2126                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2127                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2128                                    vf_id);
2129                         return -EINVAL;
2130                 }
2131
2132                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2133                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2134                 bnxt_queue_sp_work(bp);
2135                 break;
2136
2137         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2138                 bnxt_async_event_process(bp,
2139                                          (struct hwrm_async_event_cmpl *)txcmp);
2140
2141         default:
2142                 break;
2143         }
2144
2145         return 0;
2146 }
2147
2148 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2149 {
2150         struct bnxt_napi *bnapi = dev_instance;
2151         struct bnxt *bp = bnapi->bp;
2152         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2153         u32 cons = RING_CMP(cpr->cp_raw_cons);
2154
2155         cpr->event_ctr++;
2156         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2157         napi_schedule(&bnapi->napi);
2158         return IRQ_HANDLED;
2159 }
2160
2161 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2162 {
2163         u32 raw_cons = cpr->cp_raw_cons;
2164         u16 cons = RING_CMP(raw_cons);
2165         struct tx_cmp *txcmp;
2166
2167         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2168
2169         return TX_CMP_VALID(txcmp, raw_cons);
2170 }
2171
2172 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2173 {
2174         struct bnxt_napi *bnapi = dev_instance;
2175         struct bnxt *bp = bnapi->bp;
2176         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2177         u32 cons = RING_CMP(cpr->cp_raw_cons);
2178         u32 int_status;
2179
2180         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2181
2182         if (!bnxt_has_work(bp, cpr)) {
2183                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2184                 /* return if erroneous interrupt */
2185                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2186                         return IRQ_NONE;
2187         }
2188
2189         /* disable ring IRQ */
2190         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2191
2192         /* Return here if interrupt is shared and is disabled. */
2193         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2194                 return IRQ_HANDLED;
2195
2196         napi_schedule(&bnapi->napi);
2197         return IRQ_HANDLED;
2198 }
2199
2200 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2201                             int budget)
2202 {
2203         struct bnxt_napi *bnapi = cpr->bnapi;
2204         u32 raw_cons = cpr->cp_raw_cons;
2205         u32 cons;
2206         int tx_pkts = 0;
2207         int rx_pkts = 0;
2208         u8 event = 0;
2209         struct tx_cmp *txcmp;
2210
2211         cpr->has_more_work = 0;
2212         cpr->had_work_done = 1;
2213         while (1) {
2214                 int rc;
2215
2216                 cons = RING_CMP(raw_cons);
2217                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2218
2219                 if (!TX_CMP_VALID(txcmp, raw_cons))
2220                         break;
2221
2222                 /* The valid test of the entry must be done first before
2223                  * reading any further.
2224                  */
2225                 dma_rmb();
2226                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2227                         tx_pkts++;
2228                         /* return full budget so NAPI will complete. */
2229                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2230                                 rx_pkts = budget;
2231                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2232                                 if (budget)
2233                                         cpr->has_more_work = 1;
2234                                 break;
2235                         }
2236                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2237                         if (likely(budget))
2238                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2239                         else
2240                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2241                                                            &event);
2242                         if (likely(rc >= 0))
2243                                 rx_pkts += rc;
2244                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2245                          * the NAPI budget.  Otherwise, we may potentially loop
2246                          * here forever if we consistently cannot allocate
2247                          * buffers.
2248                          */
2249                         else if (rc == -ENOMEM && budget)
2250                                 rx_pkts++;
2251                         else if (rc == -EBUSY)  /* partial completion */
2252                                 break;
2253                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2254                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2255                                     (TX_CMP_TYPE(txcmp) ==
2256                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2257                                     (TX_CMP_TYPE(txcmp) ==
2258                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2259                         bnxt_hwrm_handler(bp, txcmp);
2260                 }
2261                 raw_cons = NEXT_RAW_CMP(raw_cons);
2262
2263                 if (rx_pkts && rx_pkts == budget) {
2264                         cpr->has_more_work = 1;
2265                         break;
2266                 }
2267         }
2268
2269         if (event & BNXT_REDIRECT_EVENT)
2270                 xdp_do_flush_map();
2271
2272         if (event & BNXT_TX_EVENT) {
2273                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2274                 u16 prod = txr->tx_prod;
2275
2276                 /* Sync BD data before updating doorbell */
2277                 wmb();
2278
2279                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2280         }
2281
2282         cpr->cp_raw_cons = raw_cons;
2283         bnapi->tx_pkts += tx_pkts;
2284         bnapi->events |= event;
2285         return rx_pkts;
2286 }
2287
2288 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2289 {
2290         if (bnapi->tx_pkts) {
2291                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2292                 bnapi->tx_pkts = 0;
2293         }
2294
2295         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2296                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2297
2298                 if (bnapi->events & BNXT_AGG_EVENT)
2299                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2300                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2301         }
2302         bnapi->events = 0;
2303 }
2304
2305 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2306                           int budget)
2307 {
2308         struct bnxt_napi *bnapi = cpr->bnapi;
2309         int rx_pkts;
2310
2311         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2312
2313         /* ACK completion ring before freeing tx ring and producing new
2314          * buffers in rx/agg rings to prevent overflowing the completion
2315          * ring.
2316          */
2317         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2318
2319         __bnxt_poll_work_done(bp, bnapi);
2320         return rx_pkts;
2321 }
2322
2323 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2324 {
2325         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2326         struct bnxt *bp = bnapi->bp;
2327         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2328         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2329         struct tx_cmp *txcmp;
2330         struct rx_cmp_ext *rxcmp1;
2331         u32 cp_cons, tmp_raw_cons;
2332         u32 raw_cons = cpr->cp_raw_cons;
2333         u32 rx_pkts = 0;
2334         u8 event = 0;
2335
2336         while (1) {
2337                 int rc;
2338
2339                 cp_cons = RING_CMP(raw_cons);
2340                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2341
2342                 if (!TX_CMP_VALID(txcmp, raw_cons))
2343                         break;
2344
2345                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2346                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2347                         cp_cons = RING_CMP(tmp_raw_cons);
2348                         rxcmp1 = (struct rx_cmp_ext *)
2349                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2350
2351                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2352                                 break;
2353
2354                         /* force an error to recycle the buffer */
2355                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2356                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2357
2358                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2359                         if (likely(rc == -EIO) && budget)
2360                                 rx_pkts++;
2361                         else if (rc == -EBUSY)  /* partial completion */
2362                                 break;
2363                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2364                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2365                         bnxt_hwrm_handler(bp, txcmp);
2366                 } else {
2367                         netdev_err(bp->dev,
2368                                    "Invalid completion received on special ring\n");
2369                 }
2370                 raw_cons = NEXT_RAW_CMP(raw_cons);
2371
2372                 if (rx_pkts == budget)
2373                         break;
2374         }
2375
2376         cpr->cp_raw_cons = raw_cons;
2377         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2378         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2379
2380         if (event & BNXT_AGG_EVENT)
2381                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2382
2383         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2384                 napi_complete_done(napi, rx_pkts);
2385                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2386         }
2387         return rx_pkts;
2388 }
2389
2390 static int bnxt_poll(struct napi_struct *napi, int budget)
2391 {
2392         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2393         struct bnxt *bp = bnapi->bp;
2394         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2395         int work_done = 0;
2396
2397         while (1) {
2398                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2399
2400                 if (work_done >= budget) {
2401                         if (!budget)
2402                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2403                         break;
2404                 }
2405
2406                 if (!bnxt_has_work(bp, cpr)) {
2407                         if (napi_complete_done(napi, work_done))
2408                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2409                         break;
2410                 }
2411         }
2412         if (bp->flags & BNXT_FLAG_DIM) {
2413                 struct dim_sample dim_sample = {};
2414
2415                 dim_update_sample(cpr->event_ctr,
2416                                   cpr->rx_packets,
2417                                   cpr->rx_bytes,
2418                                   &dim_sample);
2419                 net_dim(&cpr->dim, dim_sample);
2420         }
2421         return work_done;
2422 }
2423
2424 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2425 {
2426         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2427         int i, work_done = 0;
2428
2429         for (i = 0; i < 2; i++) {
2430                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2431
2432                 if (cpr2) {
2433                         work_done += __bnxt_poll_work(bp, cpr2,
2434                                                       budget - work_done);
2435                         cpr->has_more_work |= cpr2->has_more_work;
2436                 }
2437         }
2438         return work_done;
2439 }
2440
2441 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2442                                  u64 dbr_type)
2443 {
2444         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2445         int i;
2446
2447         for (i = 0; i < 2; i++) {
2448                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2449                 struct bnxt_db_info *db;
2450
2451                 if (cpr2 && cpr2->had_work_done) {
2452                         db = &cpr2->cp_db;
2453                         writeq(db->db_key64 | dbr_type |
2454                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2455                         cpr2->had_work_done = 0;
2456                 }
2457         }
2458         __bnxt_poll_work_done(bp, bnapi);
2459 }
2460
2461 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2462 {
2463         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2464         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465         u32 raw_cons = cpr->cp_raw_cons;
2466         struct bnxt *bp = bnapi->bp;
2467         struct nqe_cn *nqcmp;
2468         int work_done = 0;
2469         u32 cons;
2470
2471         if (cpr->has_more_work) {
2472                 cpr->has_more_work = 0;
2473                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2474         }
2475         while (1) {
2476                 cons = RING_CMP(raw_cons);
2477                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2478
2479                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2480                         if (cpr->has_more_work)
2481                                 break;
2482
2483                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2484                         cpr->cp_raw_cons = raw_cons;
2485                         if (napi_complete_done(napi, work_done))
2486                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2487                                                   cpr->cp_raw_cons);
2488                         return work_done;
2489                 }
2490
2491                 /* The valid test of the entry must be done first before
2492                  * reading any further.
2493                  */
2494                 dma_rmb();
2495
2496                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2497                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2498                         struct bnxt_cp_ring_info *cpr2;
2499
2500                         cpr2 = cpr->cp_ring_arr[idx];
2501                         work_done += __bnxt_poll_work(bp, cpr2,
2502                                                       budget - work_done);
2503                         cpr->has_more_work |= cpr2->has_more_work;
2504                 } else {
2505                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2506                 }
2507                 raw_cons = NEXT_RAW_CMP(raw_cons);
2508         }
2509         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2510         if (raw_cons != cpr->cp_raw_cons) {
2511                 cpr->cp_raw_cons = raw_cons;
2512                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2513         }
2514         return work_done;
2515 }
2516
2517 static void bnxt_free_tx_skbs(struct bnxt *bp)
2518 {
2519         int i, max_idx;
2520         struct pci_dev *pdev = bp->pdev;
2521
2522         if (!bp->tx_ring)
2523                 return;
2524
2525         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2526         for (i = 0; i < bp->tx_nr_rings; i++) {
2527                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2528                 int j;
2529
2530                 for (j = 0; j < max_idx;) {
2531                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2532                         struct sk_buff *skb;
2533                         int k, last;
2534
2535                         if (i < bp->tx_nr_rings_xdp &&
2536                             tx_buf->action == XDP_REDIRECT) {
2537                                 dma_unmap_single(&pdev->dev,
2538                                         dma_unmap_addr(tx_buf, mapping),
2539                                         dma_unmap_len(tx_buf, len),
2540                                         PCI_DMA_TODEVICE);
2541                                 xdp_return_frame(tx_buf->xdpf);
2542                                 tx_buf->action = 0;
2543                                 tx_buf->xdpf = NULL;
2544                                 j++;
2545                                 continue;
2546                         }
2547
2548                         skb = tx_buf->skb;
2549                         if (!skb) {
2550                                 j++;
2551                                 continue;
2552                         }
2553
2554                         tx_buf->skb = NULL;
2555
2556                         if (tx_buf->is_push) {
2557                                 dev_kfree_skb(skb);
2558                                 j += 2;
2559                                 continue;
2560                         }
2561
2562                         dma_unmap_single(&pdev->dev,
2563                                          dma_unmap_addr(tx_buf, mapping),
2564                                          skb_headlen(skb),
2565                                          PCI_DMA_TODEVICE);
2566
2567                         last = tx_buf->nr_frags;
2568                         j += 2;
2569                         for (k = 0; k < last; k++, j++) {
2570                                 int ring_idx = j & bp->tx_ring_mask;
2571                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2572
2573                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2574                                 dma_unmap_page(
2575                                         &pdev->dev,
2576                                         dma_unmap_addr(tx_buf, mapping),
2577                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2578                         }
2579                         dev_kfree_skb(skb);
2580                 }
2581                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2582         }
2583 }
2584
2585 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2586 {
2587         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2588         struct pci_dev *pdev = bp->pdev;
2589         struct bnxt_tpa_idx_map *map;
2590         int i, max_idx, max_agg_idx;
2591
2592         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2593         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2594         if (!rxr->rx_tpa)
2595                 goto skip_rx_tpa_free;
2596
2597         for (i = 0; i < bp->max_tpa; i++) {
2598                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2599                 u8 *data = tpa_info->data;
2600
2601                 if (!data)
2602                         continue;
2603
2604                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2605                                        bp->rx_buf_use_size, bp->rx_dir,
2606                                        DMA_ATTR_WEAK_ORDERING);
2607
2608                 tpa_info->data = NULL;
2609
2610                 kfree(data);
2611         }
2612
2613 skip_rx_tpa_free:
2614         for (i = 0; i < max_idx; i++) {
2615                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2616                 dma_addr_t mapping = rx_buf->mapping;
2617                 void *data = rx_buf->data;
2618
2619                 if (!data)
2620                         continue;
2621
2622                 rx_buf->data = NULL;
2623                 if (BNXT_RX_PAGE_MODE(bp)) {
2624                         mapping -= bp->rx_dma_offset;
2625                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2626                                              bp->rx_dir,
2627                                              DMA_ATTR_WEAK_ORDERING);
2628                         page_pool_recycle_direct(rxr->page_pool, data);
2629                 } else {
2630                         dma_unmap_single_attrs(&pdev->dev, mapping,
2631                                                bp->rx_buf_use_size, bp->rx_dir,
2632                                                DMA_ATTR_WEAK_ORDERING);
2633                         kfree(data);
2634                 }
2635         }
2636         for (i = 0; i < max_agg_idx; i++) {
2637                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2638                 struct page *page = rx_agg_buf->page;
2639
2640                 if (!page)
2641                         continue;
2642
2643                 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2644                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2645                                      DMA_ATTR_WEAK_ORDERING);
2646
2647                 rx_agg_buf->page = NULL;
2648                 __clear_bit(i, rxr->rx_agg_bmap);
2649
2650                 __free_page(page);
2651         }
2652         if (rxr->rx_page) {
2653                 __free_page(rxr->rx_page);
2654                 rxr->rx_page = NULL;
2655         }
2656         map = rxr->rx_tpa_idx_map;
2657         if (map)
2658                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2659 }
2660
2661 static void bnxt_free_rx_skbs(struct bnxt *bp)
2662 {
2663         int i;
2664
2665         if (!bp->rx_ring)
2666                 return;
2667
2668         for (i = 0; i < bp->rx_nr_rings; i++)
2669                 bnxt_free_one_rx_ring_skbs(bp, i);
2670 }
2671
2672 static void bnxt_free_skbs(struct bnxt *bp)
2673 {
2674         bnxt_free_tx_skbs(bp);
2675         bnxt_free_rx_skbs(bp);
2676 }
2677
2678 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2679 {
2680         struct pci_dev *pdev = bp->pdev;
2681         int i;
2682
2683         for (i = 0; i < rmem->nr_pages; i++) {
2684                 if (!rmem->pg_arr[i])
2685                         continue;
2686
2687                 dma_free_coherent(&pdev->dev, rmem->page_size,
2688                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2689
2690                 rmem->pg_arr[i] = NULL;
2691         }
2692         if (rmem->pg_tbl) {
2693                 size_t pg_tbl_size = rmem->nr_pages * 8;
2694
2695                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2696                         pg_tbl_size = rmem->page_size;
2697                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2698                                   rmem->pg_tbl, rmem->pg_tbl_map);
2699                 rmem->pg_tbl = NULL;
2700         }
2701         if (rmem->vmem_size && *rmem->vmem) {
2702                 vfree(*rmem->vmem);
2703                 *rmem->vmem = NULL;
2704         }
2705 }
2706
2707 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2708 {
2709         struct pci_dev *pdev = bp->pdev;
2710         u64 valid_bit = 0;
2711         int i;
2712
2713         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2714                 valid_bit = PTU_PTE_VALID;
2715         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2716                 size_t pg_tbl_size = rmem->nr_pages * 8;
2717
2718                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2719                         pg_tbl_size = rmem->page_size;
2720                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2721                                                   &rmem->pg_tbl_map,
2722                                                   GFP_KERNEL);
2723                 if (!rmem->pg_tbl)
2724                         return -ENOMEM;
2725         }
2726
2727         for (i = 0; i < rmem->nr_pages; i++) {
2728                 u64 extra_bits = valid_bit;
2729
2730                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2731                                                      rmem->page_size,
2732                                                      &rmem->dma_arr[i],
2733                                                      GFP_KERNEL);
2734                 if (!rmem->pg_arr[i])
2735                         return -ENOMEM;
2736
2737                 if (rmem->init_val)
2738                         memset(rmem->pg_arr[i], rmem->init_val,
2739                                rmem->page_size);
2740                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2741                         if (i == rmem->nr_pages - 2 &&
2742                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2743                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2744                         else if (i == rmem->nr_pages - 1 &&
2745                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2746                                 extra_bits |= PTU_PTE_LAST;
2747                         rmem->pg_tbl[i] =
2748                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2749                 }
2750         }
2751
2752         if (rmem->vmem_size) {
2753                 *rmem->vmem = vzalloc(rmem->vmem_size);
2754                 if (!(*rmem->vmem))
2755                         return -ENOMEM;
2756         }
2757         return 0;
2758 }
2759
2760 static void bnxt_free_tpa_info(struct bnxt *bp)
2761 {
2762         int i;
2763
2764         for (i = 0; i < bp->rx_nr_rings; i++) {
2765                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2766
2767                 kfree(rxr->rx_tpa_idx_map);
2768                 rxr->rx_tpa_idx_map = NULL;
2769                 if (rxr->rx_tpa) {
2770                         kfree(rxr->rx_tpa[0].agg_arr);
2771                         rxr->rx_tpa[0].agg_arr = NULL;
2772                 }
2773                 kfree(rxr->rx_tpa);
2774                 rxr->rx_tpa = NULL;
2775         }
2776 }
2777
2778 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2779 {
2780         int i, j, total_aggs = 0;
2781
2782         bp->max_tpa = MAX_TPA;
2783         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2784                 if (!bp->max_tpa_v2)
2785                         return 0;
2786                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2787                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2788         }
2789
2790         for (i = 0; i < bp->rx_nr_rings; i++) {
2791                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2792                 struct rx_agg_cmp *agg;
2793
2794                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2795                                       GFP_KERNEL);
2796                 if (!rxr->rx_tpa)
2797                         return -ENOMEM;
2798
2799                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2800                         continue;
2801                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2802                 rxr->rx_tpa[0].agg_arr = agg;
2803                 if (!agg)
2804                         return -ENOMEM;
2805                 for (j = 1; j < bp->max_tpa; j++)
2806                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2807                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2808                                               GFP_KERNEL);
2809                 if (!rxr->rx_tpa_idx_map)
2810                         return -ENOMEM;
2811         }
2812         return 0;
2813 }
2814
2815 static void bnxt_free_rx_rings(struct bnxt *bp)
2816 {
2817         int i;
2818
2819         if (!bp->rx_ring)
2820                 return;
2821
2822         bnxt_free_tpa_info(bp);
2823         for (i = 0; i < bp->rx_nr_rings; i++) {
2824                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2825                 struct bnxt_ring_struct *ring;
2826
2827                 if (rxr->xdp_prog)
2828                         bpf_prog_put(rxr->xdp_prog);
2829
2830                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2831                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2832
2833                 page_pool_destroy(rxr->page_pool);
2834                 rxr->page_pool = NULL;
2835
2836                 kfree(rxr->rx_agg_bmap);
2837                 rxr->rx_agg_bmap = NULL;
2838
2839                 ring = &rxr->rx_ring_struct;
2840                 bnxt_free_ring(bp, &ring->ring_mem);
2841
2842                 ring = &rxr->rx_agg_ring_struct;
2843                 bnxt_free_ring(bp, &ring->ring_mem);
2844         }
2845 }
2846
2847 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2848                                    struct bnxt_rx_ring_info *rxr)
2849 {
2850         struct page_pool_params pp = { 0 };
2851
2852         pp.pool_size = bp->rx_ring_size;
2853         pp.nid = dev_to_node(&bp->pdev->dev);
2854         pp.dev = &bp->pdev->dev;
2855         pp.dma_dir = DMA_BIDIRECTIONAL;
2856
2857         rxr->page_pool = page_pool_create(&pp);
2858         if (IS_ERR(rxr->page_pool)) {
2859                 int err = PTR_ERR(rxr->page_pool);
2860
2861                 rxr->page_pool = NULL;
2862                 return err;
2863         }
2864         return 0;
2865 }
2866
2867 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2868 {
2869         int i, rc = 0, agg_rings = 0;
2870
2871         if (!bp->rx_ring)
2872                 return -ENOMEM;
2873
2874         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2875                 agg_rings = 1;
2876
2877         for (i = 0; i < bp->rx_nr_rings; i++) {
2878                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2879                 struct bnxt_ring_struct *ring;
2880
2881                 ring = &rxr->rx_ring_struct;
2882
2883                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2884                 if (rc)
2885                         return rc;
2886
2887                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2888                 if (rc < 0)
2889                         return rc;
2890
2891                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2892                                                 MEM_TYPE_PAGE_POOL,
2893                                                 rxr->page_pool);
2894                 if (rc) {
2895                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2896                         return rc;
2897                 }
2898
2899                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2900                 if (rc)
2901                         return rc;
2902
2903                 ring->grp_idx = i;
2904                 if (agg_rings) {
2905                         u16 mem_size;
2906
2907                         ring = &rxr->rx_agg_ring_struct;
2908                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2909                         if (rc)
2910                                 return rc;
2911
2912                         ring->grp_idx = i;
2913                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2914                         mem_size = rxr->rx_agg_bmap_size / 8;
2915                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2916                         if (!rxr->rx_agg_bmap)
2917                                 return -ENOMEM;
2918                 }
2919         }
2920         if (bp->flags & BNXT_FLAG_TPA)
2921                 rc = bnxt_alloc_tpa_info(bp);
2922         return rc;
2923 }
2924
2925 static void bnxt_free_tx_rings(struct bnxt *bp)
2926 {
2927         int i;
2928         struct pci_dev *pdev = bp->pdev;
2929
2930         if (!bp->tx_ring)
2931                 return;
2932
2933         for (i = 0; i < bp->tx_nr_rings; i++) {
2934                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2935                 struct bnxt_ring_struct *ring;
2936
2937                 if (txr->tx_push) {
2938                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2939                                           txr->tx_push, txr->tx_push_mapping);
2940                         txr->tx_push = NULL;
2941                 }
2942
2943                 ring = &txr->tx_ring_struct;
2944
2945                 bnxt_free_ring(bp, &ring->ring_mem);
2946         }
2947 }
2948
2949 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2950 {
2951         int i, j, rc;
2952         struct pci_dev *pdev = bp->pdev;
2953
2954         bp->tx_push_size = 0;
2955         if (bp->tx_push_thresh) {
2956                 int push_size;
2957
2958                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2959                                         bp->tx_push_thresh);
2960
2961                 if (push_size > 256) {
2962                         push_size = 0;
2963                         bp->tx_push_thresh = 0;
2964                 }
2965
2966                 bp->tx_push_size = push_size;
2967         }
2968
2969         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2970                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2971                 struct bnxt_ring_struct *ring;
2972                 u8 qidx;
2973
2974                 ring = &txr->tx_ring_struct;
2975
2976                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2977                 if (rc)
2978                         return rc;
2979
2980                 ring->grp_idx = txr->bnapi->index;
2981                 if (bp->tx_push_size) {
2982                         dma_addr_t mapping;
2983
2984                         /* One pre-allocated DMA buffer to backup
2985                          * TX push operation
2986                          */
2987                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2988                                                 bp->tx_push_size,
2989                                                 &txr->tx_push_mapping,
2990                                                 GFP_KERNEL);
2991
2992                         if (!txr->tx_push)
2993                                 return -ENOMEM;
2994
2995                         mapping = txr->tx_push_mapping +
2996                                 sizeof(struct tx_push_bd);
2997                         txr->data_mapping = cpu_to_le64(mapping);
2998                 }
2999                 qidx = bp->tc_to_qidx[j];
3000                 ring->queue_id = bp->q_info[qidx].queue_id;
3001                 if (i < bp->tx_nr_rings_xdp)
3002                         continue;
3003                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3004                         j++;
3005         }
3006         return 0;
3007 }
3008
3009 static void bnxt_free_cp_rings(struct bnxt *bp)
3010 {
3011         int i;
3012
3013         if (!bp->bnapi)
3014                 return;
3015
3016         for (i = 0; i < bp->cp_nr_rings; i++) {
3017                 struct bnxt_napi *bnapi = bp->bnapi[i];
3018                 struct bnxt_cp_ring_info *cpr;
3019                 struct bnxt_ring_struct *ring;
3020                 int j;
3021
3022                 if (!bnapi)
3023                         continue;
3024
3025                 cpr = &bnapi->cp_ring;
3026                 ring = &cpr->cp_ring_struct;
3027
3028                 bnxt_free_ring(bp, &ring->ring_mem);
3029
3030                 for (j = 0; j < 2; j++) {
3031                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3032
3033                         if (cpr2) {
3034                                 ring = &cpr2->cp_ring_struct;
3035                                 bnxt_free_ring(bp, &ring->ring_mem);
3036                                 kfree(cpr2);
3037                                 cpr->cp_ring_arr[j] = NULL;
3038                         }
3039                 }
3040         }
3041 }
3042
3043 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3044 {
3045         struct bnxt_ring_mem_info *rmem;
3046         struct bnxt_ring_struct *ring;
3047         struct bnxt_cp_ring_info *cpr;
3048         int rc;
3049
3050         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3051         if (!cpr)
3052                 return NULL;
3053
3054         ring = &cpr->cp_ring_struct;
3055         rmem = &ring->ring_mem;
3056         rmem->nr_pages = bp->cp_nr_pages;
3057         rmem->page_size = HW_CMPD_RING_SIZE;
3058         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3059         rmem->dma_arr = cpr->cp_desc_mapping;
3060         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3061         rc = bnxt_alloc_ring(bp, rmem);
3062         if (rc) {
3063                 bnxt_free_ring(bp, rmem);
3064                 kfree(cpr);
3065                 cpr = NULL;
3066         }
3067         return cpr;
3068 }
3069
3070 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3071 {
3072         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3073         int i, rc, ulp_base_vec, ulp_msix;
3074
3075         ulp_msix = bnxt_get_ulp_msix_num(bp);
3076         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3077         for (i = 0; i < bp->cp_nr_rings; i++) {
3078                 struct bnxt_napi *bnapi = bp->bnapi[i];
3079                 struct bnxt_cp_ring_info *cpr;
3080                 struct bnxt_ring_struct *ring;
3081
3082                 if (!bnapi)
3083                         continue;
3084
3085                 cpr = &bnapi->cp_ring;
3086                 cpr->bnapi = bnapi;
3087                 ring = &cpr->cp_ring_struct;
3088
3089                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3090                 if (rc)
3091                         return rc;
3092
3093                 if (ulp_msix && i >= ulp_base_vec)
3094                         ring->map_idx = i + ulp_msix;
3095                 else
3096                         ring->map_idx = i;
3097
3098                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3099                         continue;
3100
3101                 if (i < bp->rx_nr_rings) {
3102                         struct bnxt_cp_ring_info *cpr2 =
3103                                 bnxt_alloc_cp_sub_ring(bp);
3104
3105                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3106                         if (!cpr2)
3107                                 return -ENOMEM;
3108                         cpr2->bnapi = bnapi;
3109                 }
3110                 if ((sh && i < bp->tx_nr_rings) ||
3111                     (!sh && i >= bp->rx_nr_rings)) {
3112                         struct bnxt_cp_ring_info *cpr2 =
3113                                 bnxt_alloc_cp_sub_ring(bp);
3114
3115                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3116                         if (!cpr2)
3117                                 return -ENOMEM;
3118                         cpr2->bnapi = bnapi;
3119                 }
3120         }
3121         return 0;
3122 }
3123
3124 static void bnxt_init_ring_struct(struct bnxt *bp)
3125 {
3126         int i;
3127
3128         for (i = 0; i < bp->cp_nr_rings; i++) {
3129                 struct bnxt_napi *bnapi = bp->bnapi[i];
3130                 struct bnxt_ring_mem_info *rmem;
3131                 struct bnxt_cp_ring_info *cpr;
3132                 struct bnxt_rx_ring_info *rxr;
3133                 struct bnxt_tx_ring_info *txr;
3134                 struct bnxt_ring_struct *ring;
3135
3136                 if (!bnapi)
3137                         continue;
3138
3139                 cpr = &bnapi->cp_ring;
3140                 ring = &cpr->cp_ring_struct;
3141                 rmem = &ring->ring_mem;
3142                 rmem->nr_pages = bp->cp_nr_pages;
3143                 rmem->page_size = HW_CMPD_RING_SIZE;
3144                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3145                 rmem->dma_arr = cpr->cp_desc_mapping;
3146                 rmem->vmem_size = 0;
3147
3148                 rxr = bnapi->rx_ring;
3149                 if (!rxr)
3150                         goto skip_rx;
3151
3152                 ring = &rxr->rx_ring_struct;
3153                 rmem = &ring->ring_mem;
3154                 rmem->nr_pages = bp->rx_nr_pages;
3155                 rmem->page_size = HW_RXBD_RING_SIZE;
3156                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3157                 rmem->dma_arr = rxr->rx_desc_mapping;
3158                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3159                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3160
3161                 ring = &rxr->rx_agg_ring_struct;
3162                 rmem = &ring->ring_mem;
3163                 rmem->nr_pages = bp->rx_agg_nr_pages;
3164                 rmem->page_size = HW_RXBD_RING_SIZE;
3165                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3166                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3167                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3168                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3169
3170 skip_rx:
3171                 txr = bnapi->tx_ring;
3172                 if (!txr)
3173                         continue;
3174
3175                 ring = &txr->tx_ring_struct;
3176                 rmem = &ring->ring_mem;
3177                 rmem->nr_pages = bp->tx_nr_pages;
3178                 rmem->page_size = HW_RXBD_RING_SIZE;
3179                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3180                 rmem->dma_arr = txr->tx_desc_mapping;
3181                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3182                 rmem->vmem = (void **)&txr->tx_buf_ring;
3183         }
3184 }
3185
3186 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3187 {
3188         int i;
3189         u32 prod;
3190         struct rx_bd **rx_buf_ring;
3191
3192         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3193         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3194                 int j;
3195                 struct rx_bd *rxbd;
3196
3197                 rxbd = rx_buf_ring[i];
3198                 if (!rxbd)
3199                         continue;
3200
3201                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3202                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3203                         rxbd->rx_bd_opaque = prod;
3204                 }
3205         }
3206 }
3207
3208 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3209 {
3210         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3211         struct net_device *dev = bp->dev;
3212         u32 prod;
3213         int i;
3214
3215         prod = rxr->rx_prod;
3216         for (i = 0; i < bp->rx_ring_size; i++) {
3217                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3218                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3219                                     ring_nr, i, bp->rx_ring_size);
3220                         break;
3221                 }
3222                 prod = NEXT_RX(prod);
3223         }
3224         rxr->rx_prod = prod;
3225
3226         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3227                 return 0;
3228
3229         prod = rxr->rx_agg_prod;
3230         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3231                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3232                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3233                                     ring_nr, i, bp->rx_ring_size);
3234                         break;
3235                 }
3236                 prod = NEXT_RX_AGG(prod);
3237         }
3238         rxr->rx_agg_prod = prod;
3239
3240         if (rxr->rx_tpa) {
3241                 dma_addr_t mapping;
3242                 u8 *data;
3243
3244                 for (i = 0; i < bp->max_tpa; i++) {
3245                         data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3246                         if (!data)
3247                                 return -ENOMEM;
3248
3249                         rxr->rx_tpa[i].data = data;
3250                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3251                         rxr->rx_tpa[i].mapping = mapping;
3252                 }
3253         }
3254         return 0;
3255 }
3256
3257 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3258 {
3259         struct bnxt_rx_ring_info *rxr;
3260         struct bnxt_ring_struct *ring;
3261         u32 type;
3262
3263         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3264                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3265
3266         if (NET_IP_ALIGN == 2)
3267                 type |= RX_BD_FLAGS_SOP;
3268
3269         rxr = &bp->rx_ring[ring_nr];
3270         ring = &rxr->rx_ring_struct;
3271         bnxt_init_rxbd_pages(ring, type);
3272
3273         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3274                 bpf_prog_add(bp->xdp_prog, 1);
3275                 rxr->xdp_prog = bp->xdp_prog;
3276         }
3277         ring->fw_ring_id = INVALID_HW_RING_ID;
3278
3279         ring = &rxr->rx_agg_ring_struct;
3280         ring->fw_ring_id = INVALID_HW_RING_ID;
3281
3282         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3283                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3284                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3285
3286                 bnxt_init_rxbd_pages(ring, type);
3287         }
3288
3289         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3290 }
3291
3292 static void bnxt_init_cp_rings(struct bnxt *bp)
3293 {
3294         int i, j;
3295
3296         for (i = 0; i < bp->cp_nr_rings; i++) {
3297                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3298                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3299
3300                 ring->fw_ring_id = INVALID_HW_RING_ID;
3301                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3302                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3303                 for (j = 0; j < 2; j++) {
3304                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3305
3306                         if (!cpr2)
3307                                 continue;
3308
3309                         ring = &cpr2->cp_ring_struct;
3310                         ring->fw_ring_id = INVALID_HW_RING_ID;
3311                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3312                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3313                 }
3314         }
3315 }
3316
3317 static int bnxt_init_rx_rings(struct bnxt *bp)
3318 {
3319         int i, rc = 0;
3320
3321         if (BNXT_RX_PAGE_MODE(bp)) {
3322                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3323                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3324         } else {
3325                 bp->rx_offset = BNXT_RX_OFFSET;
3326                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3327         }
3328
3329         for (i = 0; i < bp->rx_nr_rings; i++) {
3330                 rc = bnxt_init_one_rx_ring(bp, i);
3331                 if (rc)
3332                         break;
3333         }
3334
3335         return rc;
3336 }
3337
3338 static int bnxt_init_tx_rings(struct bnxt *bp)
3339 {
3340         u16 i;
3341
3342         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3343                                    MAX_SKB_FRAGS + 1);
3344
3345         for (i = 0; i < bp->tx_nr_rings; i++) {
3346                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3347                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3348
3349                 ring->fw_ring_id = INVALID_HW_RING_ID;
3350         }
3351
3352         return 0;
3353 }
3354
3355 static void bnxt_free_ring_grps(struct bnxt *bp)
3356 {
3357         kfree(bp->grp_info);
3358         bp->grp_info = NULL;
3359 }
3360
3361 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3362 {
3363         int i;
3364
3365         if (irq_re_init) {
3366                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3367                                        sizeof(struct bnxt_ring_grp_info),
3368                                        GFP_KERNEL);
3369                 if (!bp->grp_info)
3370                         return -ENOMEM;
3371         }
3372         for (i = 0; i < bp->cp_nr_rings; i++) {
3373                 if (irq_re_init)
3374                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3375                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3376                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3377                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3378                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3379         }
3380         return 0;
3381 }
3382
3383 static void bnxt_free_vnics(struct bnxt *bp)
3384 {
3385         kfree(bp->vnic_info);
3386         bp->vnic_info = NULL;
3387         bp->nr_vnics = 0;
3388 }
3389
3390 static int bnxt_alloc_vnics(struct bnxt *bp)
3391 {
3392         int num_vnics = 1;
3393
3394 #ifdef CONFIG_RFS_ACCEL
3395         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3396                 num_vnics += bp->rx_nr_rings;
3397 #endif
3398
3399         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3400                 num_vnics++;
3401
3402         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3403                                 GFP_KERNEL);
3404         if (!bp->vnic_info)
3405                 return -ENOMEM;
3406
3407         bp->nr_vnics = num_vnics;
3408         return 0;
3409 }
3410
3411 static void bnxt_init_vnics(struct bnxt *bp)
3412 {
3413         int i;
3414
3415         for (i = 0; i < bp->nr_vnics; i++) {
3416                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3417                 int j;
3418
3419                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3420                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3421                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3422
3423                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3424
3425                 if (bp->vnic_info[i].rss_hash_key) {
3426                         if (i == 0)
3427                                 prandom_bytes(vnic->rss_hash_key,
3428                                               HW_HASH_KEY_SIZE);
3429                         else
3430                                 memcpy(vnic->rss_hash_key,
3431                                        bp->vnic_info[0].rss_hash_key,
3432                                        HW_HASH_KEY_SIZE);
3433                 }
3434         }
3435 }
3436
3437 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3438 {
3439         int pages;
3440
3441         pages = ring_size / desc_per_pg;
3442
3443         if (!pages)
3444                 return 1;
3445
3446         pages++;
3447
3448         while (pages & (pages - 1))
3449                 pages++;
3450
3451         return pages;
3452 }
3453
3454 void bnxt_set_tpa_flags(struct bnxt *bp)
3455 {
3456         bp->flags &= ~BNXT_FLAG_TPA;
3457         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3458                 return;
3459         if (bp->dev->features & NETIF_F_LRO)
3460                 bp->flags |= BNXT_FLAG_LRO;
3461         else if (bp->dev->features & NETIF_F_GRO_HW)
3462                 bp->flags |= BNXT_FLAG_GRO;
3463 }
3464
3465 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3466  * be set on entry.
3467  */
3468 void bnxt_set_ring_params(struct bnxt *bp)
3469 {
3470         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3471         u32 agg_factor = 0, agg_ring_size = 0;
3472
3473         /* 8 for CRC and VLAN */
3474         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3475
3476         rx_space = rx_size + NET_SKB_PAD +
3477                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3478
3479         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3480         ring_size = bp->rx_ring_size;
3481         bp->rx_agg_ring_size = 0;
3482         bp->rx_agg_nr_pages = 0;
3483
3484         if (bp->flags & BNXT_FLAG_TPA)
3485                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3486
3487         bp->flags &= ~BNXT_FLAG_JUMBO;
3488         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3489                 u32 jumbo_factor;
3490
3491                 bp->flags |= BNXT_FLAG_JUMBO;
3492                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3493                 if (jumbo_factor > agg_factor)
3494                         agg_factor = jumbo_factor;
3495         }
3496         agg_ring_size = ring_size * agg_factor;
3497
3498         if (agg_ring_size) {
3499                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3500                                                         RX_DESC_CNT);
3501                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3502                         u32 tmp = agg_ring_size;
3503
3504                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3505                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3506                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3507                                     tmp, agg_ring_size);
3508                 }
3509                 bp->rx_agg_ring_size = agg_ring_size;
3510                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3511                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3512                 rx_space = rx_size + NET_SKB_PAD +
3513                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3514         }
3515
3516         bp->rx_buf_use_size = rx_size;
3517         bp->rx_buf_size = rx_space;
3518
3519         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3520         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3521
3522         ring_size = bp->tx_ring_size;
3523         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3524         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3525
3526         max_rx_cmpl = bp->rx_ring_size;
3527         /* MAX TPA needs to be added because TPA_START completions are
3528          * immediately recycled, so the TPA completions are not bound by
3529          * the RX ring size.
3530          */
3531         if (bp->flags & BNXT_FLAG_TPA)
3532                 max_rx_cmpl += bp->max_tpa;
3533         /* RX and TPA completions are 32-byte, all others are 16-byte */
3534         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3535         bp->cp_ring_size = ring_size;
3536
3537         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3538         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3539                 bp->cp_nr_pages = MAX_CP_PAGES;
3540                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3541                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3542                             ring_size, bp->cp_ring_size);
3543         }
3544         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3545         bp->cp_ring_mask = bp->cp_bit - 1;
3546 }
3547
3548 /* Changing allocation mode of RX rings.
3549  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3550  */
3551 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3552 {
3553         if (page_mode) {
3554                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3555                         return -EOPNOTSUPP;
3556                 bp->dev->max_mtu =
3557                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3558                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3559                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3560                 bp->rx_dir = DMA_BIDIRECTIONAL;
3561                 bp->rx_skb_func = bnxt_rx_page_skb;
3562                 /* Disable LRO or GRO_HW */
3563                 netdev_update_features(bp->dev);
3564         } else {
3565                 bp->dev->max_mtu = bp->max_mtu;
3566                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3567                 bp->rx_dir = DMA_FROM_DEVICE;
3568                 bp->rx_skb_func = bnxt_rx_skb;
3569         }
3570         return 0;
3571 }
3572
3573 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3574 {
3575         int i;
3576         struct bnxt_vnic_info *vnic;
3577         struct pci_dev *pdev = bp->pdev;
3578
3579         if (!bp->vnic_info)
3580                 return;
3581
3582         for (i = 0; i < bp->nr_vnics; i++) {
3583                 vnic = &bp->vnic_info[i];
3584
3585                 kfree(vnic->fw_grp_ids);
3586                 vnic->fw_grp_ids = NULL;
3587
3588                 kfree(vnic->uc_list);
3589                 vnic->uc_list = NULL;
3590
3591                 if (vnic->mc_list) {
3592                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3593                                           vnic->mc_list, vnic->mc_list_mapping);
3594                         vnic->mc_list = NULL;
3595                 }
3596
3597                 if (vnic->rss_table) {
3598                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3599                                           vnic->rss_table,
3600                                           vnic->rss_table_dma_addr);
3601                         vnic->rss_table = NULL;
3602                 }
3603
3604                 vnic->rss_hash_key = NULL;
3605                 vnic->flags = 0;
3606         }
3607 }
3608
3609 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3610 {
3611         int i, rc = 0, size;
3612         struct bnxt_vnic_info *vnic;
3613         struct pci_dev *pdev = bp->pdev;
3614         int max_rings;
3615
3616         for (i = 0; i < bp->nr_vnics; i++) {
3617                 vnic = &bp->vnic_info[i];
3618
3619                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3620                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3621
3622                         if (mem_size > 0) {
3623                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3624                                 if (!vnic->uc_list) {
3625                                         rc = -ENOMEM;
3626                                         goto out;
3627                                 }
3628                         }
3629                 }
3630
3631                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3632                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3633                         vnic->mc_list =
3634                                 dma_alloc_coherent(&pdev->dev,
3635                                                    vnic->mc_list_size,
3636                                                    &vnic->mc_list_mapping,
3637                                                    GFP_KERNEL);
3638                         if (!vnic->mc_list) {
3639                                 rc = -ENOMEM;
3640                                 goto out;
3641                         }
3642                 }
3643
3644                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3645                         goto vnic_skip_grps;
3646
3647                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3648                         max_rings = bp->rx_nr_rings;
3649                 else
3650                         max_rings = 1;
3651
3652                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3653                 if (!vnic->fw_grp_ids) {
3654                         rc = -ENOMEM;
3655                         goto out;
3656                 }
3657 vnic_skip_grps:
3658                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3659                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3660                         continue;
3661
3662                 /* Allocate rss table and hash key */
3663                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3664                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3665                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3666
3667                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3668                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3669                                                      vnic->rss_table_size,
3670                                                      &vnic->rss_table_dma_addr,
3671                                                      GFP_KERNEL);
3672                 if (!vnic->rss_table) {
3673                         rc = -ENOMEM;
3674                         goto out;
3675                 }
3676
3677                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3678                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3679         }
3680         return 0;
3681
3682 out:
3683         return rc;
3684 }
3685
3686 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3687 {
3688         struct pci_dev *pdev = bp->pdev;
3689
3690         if (bp->hwrm_cmd_resp_addr) {
3691                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3692                                   bp->hwrm_cmd_resp_dma_addr);
3693                 bp->hwrm_cmd_resp_addr = NULL;
3694         }
3695
3696         if (bp->hwrm_cmd_kong_resp_addr) {
3697                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3698                                   bp->hwrm_cmd_kong_resp_addr,
3699                                   bp->hwrm_cmd_kong_resp_dma_addr);
3700                 bp->hwrm_cmd_kong_resp_addr = NULL;
3701         }
3702 }
3703
3704 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3705 {
3706         struct pci_dev *pdev = bp->pdev;
3707
3708         if (bp->hwrm_cmd_kong_resp_addr)
3709                 return 0;
3710
3711         bp->hwrm_cmd_kong_resp_addr =
3712                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3713                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3714                                    GFP_KERNEL);
3715         if (!bp->hwrm_cmd_kong_resp_addr)
3716                 return -ENOMEM;
3717
3718         return 0;
3719 }
3720
3721 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3722 {
3723         struct pci_dev *pdev = bp->pdev;
3724
3725         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3726                                                    &bp->hwrm_cmd_resp_dma_addr,
3727                                                    GFP_KERNEL);
3728         if (!bp->hwrm_cmd_resp_addr)
3729                 return -ENOMEM;
3730
3731         return 0;
3732 }
3733
3734 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3735 {
3736         if (bp->hwrm_short_cmd_req_addr) {
3737                 struct pci_dev *pdev = bp->pdev;
3738
3739                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3740                                   bp->hwrm_short_cmd_req_addr,
3741                                   bp->hwrm_short_cmd_req_dma_addr);
3742                 bp->hwrm_short_cmd_req_addr = NULL;
3743         }
3744 }
3745
3746 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3747 {
3748         struct pci_dev *pdev = bp->pdev;
3749
3750         if (bp->hwrm_short_cmd_req_addr)
3751                 return 0;
3752
3753         bp->hwrm_short_cmd_req_addr =
3754                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3755                                    &bp->hwrm_short_cmd_req_dma_addr,
3756                                    GFP_KERNEL);
3757         if (!bp->hwrm_short_cmd_req_addr)
3758                 return -ENOMEM;
3759
3760         return 0;
3761 }
3762
3763 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3764 {
3765         kfree(stats->hw_masks);
3766         stats->hw_masks = NULL;
3767         kfree(stats->sw_stats);
3768         stats->sw_stats = NULL;
3769         if (stats->hw_stats) {
3770                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3771                                   stats->hw_stats_map);
3772                 stats->hw_stats = NULL;
3773         }
3774 }
3775
3776 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3777                                 bool alloc_masks)
3778 {
3779         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3780                                              &stats->hw_stats_map, GFP_KERNEL);
3781         if (!stats->hw_stats)
3782                 return -ENOMEM;
3783
3784         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3785         if (!stats->sw_stats)
3786                 goto stats_mem_err;
3787
3788         if (alloc_masks) {
3789                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3790                 if (!stats->hw_masks)
3791                         goto stats_mem_err;
3792         }
3793         return 0;
3794
3795 stats_mem_err:
3796         bnxt_free_stats_mem(bp, stats);
3797         return -ENOMEM;
3798 }
3799
3800 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3801 {
3802         int i;
3803
3804         for (i = 0; i < count; i++)
3805                 mask_arr[i] = mask;
3806 }
3807
3808 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3809 {
3810         int i;
3811
3812         for (i = 0; i < count; i++)
3813                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3814 }
3815
3816 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3817                                     struct bnxt_stats_mem *stats)
3818 {
3819         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3820         struct hwrm_func_qstats_ext_input req = {0};
3821         __le64 *hw_masks;
3822         int rc;
3823
3824         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3825             !(bp->flags & BNXT_FLAG_CHIP_P5))
3826                 return -EOPNOTSUPP;
3827
3828         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3829         req.fid = cpu_to_le16(0xffff);
3830         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3831         mutex_lock(&bp->hwrm_cmd_lock);
3832         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3833         if (rc)
3834                 goto qstat_exit;
3835
3836         hw_masks = &resp->rx_ucast_pkts;
3837         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3838
3839 qstat_exit:
3840         mutex_unlock(&bp->hwrm_cmd_lock);
3841         return rc;
3842 }
3843
3844 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3845 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3846
3847 static void bnxt_init_stats(struct bnxt *bp)
3848 {
3849         struct bnxt_napi *bnapi = bp->bnapi[0];
3850         struct bnxt_cp_ring_info *cpr;
3851         struct bnxt_stats_mem *stats;
3852         __le64 *rx_stats, *tx_stats;
3853         int rc, rx_count, tx_count;
3854         u64 *rx_masks, *tx_masks;
3855         u64 mask;
3856         u8 flags;
3857
3858         cpr = &bnapi->cp_ring;
3859         stats = &cpr->stats;
3860         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3861         if (rc) {
3862                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3863                         mask = (1ULL << 48) - 1;
3864                 else
3865                         mask = -1ULL;
3866                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3867         }
3868         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3869                 stats = &bp->port_stats;
3870                 rx_stats = stats->hw_stats;
3871                 rx_masks = stats->hw_masks;
3872                 rx_count = sizeof(struct rx_port_stats) / 8;
3873                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3874                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3875                 tx_count = sizeof(struct tx_port_stats) / 8;
3876
3877                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3878                 rc = bnxt_hwrm_port_qstats(bp, flags);
3879                 if (rc) {
3880                         mask = (1ULL << 40) - 1;
3881
3882                         bnxt_fill_masks(rx_masks, mask, rx_count);
3883                         bnxt_fill_masks(tx_masks, mask, tx_count);
3884                 } else {
3885                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3886                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3887                         bnxt_hwrm_port_qstats(bp, 0);
3888                 }
3889         }
3890         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3891                 stats = &bp->rx_port_stats_ext;
3892                 rx_stats = stats->hw_stats;
3893                 rx_masks = stats->hw_masks;
3894                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3895                 stats = &bp->tx_port_stats_ext;
3896                 tx_stats = stats->hw_stats;
3897                 tx_masks = stats->hw_masks;
3898                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3899
3900                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3901                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3902                 if (rc) {
3903                         mask = (1ULL << 40) - 1;
3904
3905                         bnxt_fill_masks(rx_masks, mask, rx_count);
3906                         if (tx_stats)
3907                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3908                 } else {
3909                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3910                         if (tx_stats)
3911                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3912                                                    tx_count);
3913                         bnxt_hwrm_port_qstats_ext(bp, 0);
3914                 }
3915         }
3916 }
3917
3918 static void bnxt_free_port_stats(struct bnxt *bp)
3919 {
3920         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3921         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3922
3923         bnxt_free_stats_mem(bp, &bp->port_stats);
3924         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3925         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3926 }
3927
3928 static void bnxt_free_ring_stats(struct bnxt *bp)
3929 {
3930         int i;
3931
3932         if (!bp->bnapi)
3933                 return;
3934
3935         for (i = 0; i < bp->cp_nr_rings; i++) {
3936                 struct bnxt_napi *bnapi = bp->bnapi[i];
3937                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3938
3939                 bnxt_free_stats_mem(bp, &cpr->stats);
3940         }
3941 }
3942
3943 static int bnxt_alloc_stats(struct bnxt *bp)
3944 {
3945         u32 size, i;
3946         int rc;
3947
3948         size = bp->hw_ring_stats_size;
3949
3950         for (i = 0; i < bp->cp_nr_rings; i++) {
3951                 struct bnxt_napi *bnapi = bp->bnapi[i];
3952                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3953
3954                 cpr->stats.len = size;
3955                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3956                 if (rc)
3957                         return rc;
3958
3959                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3960         }
3961
3962         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3963                 return 0;
3964
3965         if (bp->port_stats.hw_stats)
3966                 goto alloc_ext_stats;
3967
3968         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3969         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3970         if (rc)
3971                 return rc;
3972
3973         bp->flags |= BNXT_FLAG_PORT_STATS;
3974
3975 alloc_ext_stats:
3976         /* Display extended statistics only if FW supports it */
3977         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3978                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3979                         return 0;
3980
3981         if (bp->rx_port_stats_ext.hw_stats)
3982                 goto alloc_tx_ext_stats;
3983
3984         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3985         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3986         /* Extended stats are optional */
3987         if (rc)
3988                 return 0;
3989
3990 alloc_tx_ext_stats:
3991         if (bp->tx_port_stats_ext.hw_stats)
3992                 return 0;
3993
3994         if (bp->hwrm_spec_code >= 0x10902 ||
3995             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3996                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3997                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3998                 /* Extended stats are optional */
3999                 if (rc)
4000                         return 0;
4001         }
4002         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4003         return 0;
4004 }
4005
4006 static void bnxt_clear_ring_indices(struct bnxt *bp)
4007 {
4008         int i;
4009
4010         if (!bp->bnapi)
4011                 return;
4012
4013         for (i = 0; i < bp->cp_nr_rings; i++) {
4014                 struct bnxt_napi *bnapi = bp->bnapi[i];
4015                 struct bnxt_cp_ring_info *cpr;
4016                 struct bnxt_rx_ring_info *rxr;
4017                 struct bnxt_tx_ring_info *txr;
4018
4019                 if (!bnapi)
4020                         continue;
4021
4022                 cpr = &bnapi->cp_ring;
4023                 cpr->cp_raw_cons = 0;
4024
4025                 txr = bnapi->tx_ring;
4026                 if (txr) {
4027                         txr->tx_prod = 0;
4028                         txr->tx_cons = 0;
4029                 }
4030
4031                 rxr = bnapi->rx_ring;
4032                 if (rxr) {
4033                         rxr->rx_prod = 0;
4034                         rxr->rx_agg_prod = 0;
4035                         rxr->rx_sw_agg_prod = 0;
4036                         rxr->rx_next_cons = 0;
4037                 }
4038         }
4039 }
4040
4041 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4042 {
4043 #ifdef CONFIG_RFS_ACCEL
4044         int i;
4045
4046         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4047          * safe to delete the hash table.
4048          */
4049         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4050                 struct hlist_head *head;
4051                 struct hlist_node *tmp;
4052                 struct bnxt_ntuple_filter *fltr;
4053
4054                 head = &bp->ntp_fltr_hash_tbl[i];
4055                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4056                         hlist_del(&fltr->hash);
4057                         kfree(fltr);
4058                 }
4059         }
4060         if (irq_reinit) {
4061                 kfree(bp->ntp_fltr_bmap);
4062                 bp->ntp_fltr_bmap = NULL;
4063         }
4064         bp->ntp_fltr_count = 0;
4065 #endif
4066 }
4067
4068 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4069 {
4070 #ifdef CONFIG_RFS_ACCEL
4071         int i, rc = 0;
4072
4073         if (!(bp->flags & BNXT_FLAG_RFS))
4074                 return 0;
4075
4076         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4077                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4078
4079         bp->ntp_fltr_count = 0;
4080         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4081                                     sizeof(long),
4082                                     GFP_KERNEL);
4083
4084         if (!bp->ntp_fltr_bmap)
4085                 rc = -ENOMEM;
4086
4087         return rc;
4088 #else
4089         return 0;
4090 #endif
4091 }
4092
4093 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4094 {
4095         bnxt_free_vnic_attributes(bp);
4096         bnxt_free_tx_rings(bp);
4097         bnxt_free_rx_rings(bp);
4098         bnxt_free_cp_rings(bp);
4099         bnxt_free_ntp_fltrs(bp, irq_re_init);
4100         if (irq_re_init) {
4101                 bnxt_free_ring_stats(bp);
4102                 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) ||
4103                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4104                         bnxt_free_port_stats(bp);
4105                 bnxt_free_ring_grps(bp);
4106                 bnxt_free_vnics(bp);
4107                 kfree(bp->tx_ring_map);
4108                 bp->tx_ring_map = NULL;
4109                 kfree(bp->tx_ring);
4110                 bp->tx_ring = NULL;
4111                 kfree(bp->rx_ring);
4112                 bp->rx_ring = NULL;
4113                 kfree(bp->bnapi);
4114                 bp->bnapi = NULL;
4115         } else {
4116                 bnxt_clear_ring_indices(bp);
4117         }
4118 }
4119
4120 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4121 {
4122         int i, j, rc, size, arr_size;
4123         void *bnapi;
4124
4125         if (irq_re_init) {
4126                 /* Allocate bnapi mem pointer array and mem block for
4127                  * all queues
4128                  */
4129                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4130                                 bp->cp_nr_rings);
4131                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4132                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4133                 if (!bnapi)
4134                         return -ENOMEM;
4135
4136                 bp->bnapi = bnapi;
4137                 bnapi += arr_size;
4138                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4139                         bp->bnapi[i] = bnapi;
4140                         bp->bnapi[i]->index = i;
4141                         bp->bnapi[i]->bp = bp;
4142                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4143                                 struct bnxt_cp_ring_info *cpr =
4144                                         &bp->bnapi[i]->cp_ring;
4145
4146                                 cpr->cp_ring_struct.ring_mem.flags =
4147                                         BNXT_RMEM_RING_PTE_FLAG;
4148                         }
4149                 }
4150
4151                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4152                                       sizeof(struct bnxt_rx_ring_info),
4153                                       GFP_KERNEL);
4154                 if (!bp->rx_ring)
4155                         return -ENOMEM;
4156
4157                 for (i = 0; i < bp->rx_nr_rings; i++) {
4158                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4159
4160                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4161                                 rxr->rx_ring_struct.ring_mem.flags =
4162                                         BNXT_RMEM_RING_PTE_FLAG;
4163                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4164                                         BNXT_RMEM_RING_PTE_FLAG;
4165                         }
4166                         rxr->bnapi = bp->bnapi[i];
4167                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4168                 }
4169
4170                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4171                                       sizeof(struct bnxt_tx_ring_info),
4172                                       GFP_KERNEL);
4173                 if (!bp->tx_ring)
4174                         return -ENOMEM;
4175
4176                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4177                                           GFP_KERNEL);
4178
4179                 if (!bp->tx_ring_map)
4180                         return -ENOMEM;
4181
4182                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4183                         j = 0;
4184                 else
4185                         j = bp->rx_nr_rings;
4186
4187                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4188                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4189
4190                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4191                                 txr->tx_ring_struct.ring_mem.flags =
4192                                         BNXT_RMEM_RING_PTE_FLAG;
4193                         txr->bnapi = bp->bnapi[j];
4194                         bp->bnapi[j]->tx_ring = txr;
4195                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4196                         if (i >= bp->tx_nr_rings_xdp) {
4197                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4198                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4199                         } else {
4200                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4201                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4202                         }
4203                 }
4204
4205                 rc = bnxt_alloc_stats(bp);
4206                 if (rc)
4207                         goto alloc_mem_err;
4208                 bnxt_init_stats(bp);
4209
4210                 rc = bnxt_alloc_ntp_fltrs(bp);
4211                 if (rc)
4212                         goto alloc_mem_err;
4213
4214                 rc = bnxt_alloc_vnics(bp);
4215                 if (rc)
4216                         goto alloc_mem_err;
4217         }
4218
4219         bnxt_init_ring_struct(bp);
4220
4221         rc = bnxt_alloc_rx_rings(bp);
4222         if (rc)
4223                 goto alloc_mem_err;
4224
4225         rc = bnxt_alloc_tx_rings(bp);
4226         if (rc)
4227                 goto alloc_mem_err;
4228
4229         rc = bnxt_alloc_cp_rings(bp);
4230         if (rc)
4231                 goto alloc_mem_err;
4232
4233         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4234                                   BNXT_VNIC_UCAST_FLAG;
4235         rc = bnxt_alloc_vnic_attributes(bp);
4236         if (rc)
4237                 goto alloc_mem_err;
4238         return 0;
4239
4240 alloc_mem_err:
4241         bnxt_free_mem(bp, true);
4242         return rc;
4243 }
4244
4245 static void bnxt_disable_int(struct bnxt *bp)
4246 {
4247         int i;
4248
4249         if (!bp->bnapi)
4250                 return;
4251
4252         for (i = 0; i < bp->cp_nr_rings; i++) {
4253                 struct bnxt_napi *bnapi = bp->bnapi[i];
4254                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4255                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4256
4257                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4258                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4259         }
4260 }
4261
4262 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4263 {
4264         struct bnxt_napi *bnapi = bp->bnapi[n];
4265         struct bnxt_cp_ring_info *cpr;
4266
4267         cpr = &bnapi->cp_ring;
4268         return cpr->cp_ring_struct.map_idx;
4269 }
4270
4271 static void bnxt_disable_int_sync(struct bnxt *bp)
4272 {
4273         int i;
4274
4275         atomic_inc(&bp->intr_sem);
4276
4277         bnxt_disable_int(bp);
4278         for (i = 0; i < bp->cp_nr_rings; i++) {
4279                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4280
4281                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4282         }
4283 }
4284
4285 static void bnxt_enable_int(struct bnxt *bp)
4286 {
4287         int i;
4288
4289         atomic_set(&bp->intr_sem, 0);
4290         for (i = 0; i < bp->cp_nr_rings; i++) {
4291                 struct bnxt_napi *bnapi = bp->bnapi[i];
4292                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4293
4294                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4295         }
4296 }
4297
4298 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4299                             u16 cmpl_ring, u16 target_id)
4300 {
4301         struct input *req = request;
4302
4303         req->req_type = cpu_to_le16(req_type);
4304         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4305         req->target_id = cpu_to_le16(target_id);
4306         if (bnxt_kong_hwrm_message(bp, req))
4307                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4308         else
4309                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4310 }
4311
4312 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4313 {
4314         switch (hwrm_err) {
4315         case HWRM_ERR_CODE_SUCCESS:
4316                 return 0;
4317         case HWRM_ERR_CODE_RESOURCE_LOCKED:
4318                 return -EROFS;
4319         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4320                 return -EACCES;
4321         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4322                 return -ENOSPC;
4323         case HWRM_ERR_CODE_INVALID_PARAMS:
4324         case HWRM_ERR_CODE_INVALID_FLAGS:
4325         case HWRM_ERR_CODE_INVALID_ENABLES:
4326         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4327         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4328                 return -EINVAL;
4329         case HWRM_ERR_CODE_NO_BUFFER:
4330                 return -ENOMEM;
4331         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4332         case HWRM_ERR_CODE_BUSY:
4333                 return -EAGAIN;
4334         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4335                 return -EOPNOTSUPP;
4336         default:
4337                 return -EIO;
4338         }
4339 }
4340
4341 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4342                                  int timeout, bool silent)
4343 {
4344         int i, intr_process, rc, tmo_count;
4345         struct input *req = msg;
4346         u32 *data = msg;
4347         u8 *valid;
4348         u16 cp_ring_id, len = 0;
4349         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4350         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4351         struct hwrm_short_input short_input = {0};
4352         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4353         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4354         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4355
4356         if (BNXT_NO_FW_ACCESS(bp) &&
4357             le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4358                 return -EBUSY;
4359
4360         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4361                 if (msg_len > bp->hwrm_max_ext_req_len ||
4362                     !bp->hwrm_short_cmd_req_addr)
4363                         return -EINVAL;
4364         }
4365
4366         if (bnxt_hwrm_kong_chnl(bp, req)) {
4367                 dst = BNXT_HWRM_CHNL_KONG;
4368                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4369                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4370                 resp = bp->hwrm_cmd_kong_resp_addr;
4371         }
4372
4373         memset(resp, 0, PAGE_SIZE);
4374         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4375         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4376
4377         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4378         /* currently supports only one outstanding message */
4379         if (intr_process)
4380                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4381
4382         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4383             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4384                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4385                 u16 max_msg_len;
4386
4387                 /* Set boundary for maximum extended request length for short
4388                  * cmd format. If passed up from device use the max supported
4389                  * internal req length.
4390                  */
4391                 max_msg_len = bp->hwrm_max_ext_req_len;
4392
4393                 memcpy(short_cmd_req, req, msg_len);
4394                 if (msg_len < max_msg_len)
4395                         memset(short_cmd_req + msg_len, 0,
4396                                max_msg_len - msg_len);
4397
4398                 short_input.req_type = req->req_type;
4399                 short_input.signature =
4400                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4401                 short_input.size = cpu_to_le16(msg_len);
4402                 short_input.req_addr =
4403                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4404
4405                 data = (u32 *)&short_input;
4406                 msg_len = sizeof(short_input);
4407
4408                 /* Sync memory write before updating doorbell */
4409                 wmb();
4410
4411                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4412         }
4413
4414         /* Write request msg to hwrm channel */
4415         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4416
4417         for (i = msg_len; i < max_req_len; i += 4)
4418                 writel(0, bp->bar0 + bar_offset + i);
4419
4420         /* Ring channel doorbell */
4421         writel(1, bp->bar0 + doorbell_offset);
4422
4423         if (!pci_is_enabled(bp->pdev))
4424                 return 0;
4425
4426         if (!timeout)
4427                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4428         /* convert timeout to usec */
4429         timeout *= 1000;
4430
4431         i = 0;
4432         /* Short timeout for the first few iterations:
4433          * number of loops = number of loops for short timeout +
4434          * number of loops for standard timeout.
4435          */
4436         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4437         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4438         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4439
4440         if (intr_process) {
4441                 u16 seq_id = bp->hwrm_intr_seq_id;
4442
4443                 /* Wait until hwrm response cmpl interrupt is processed */
4444                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4445                        i++ < tmo_count) {
4446                         /* Abort the wait for completion if the FW health
4447                          * check has failed.
4448                          */
4449                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4450                                 return -EBUSY;
4451                         /* on first few passes, just barely sleep */
4452                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4453                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4454                                              HWRM_SHORT_MAX_TIMEOUT);
4455                         else
4456                                 usleep_range(HWRM_MIN_TIMEOUT,
4457                                              HWRM_MAX_TIMEOUT);
4458                 }
4459
4460                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4461                         if (!silent)
4462                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4463                                            le16_to_cpu(req->req_type));
4464                         return -EBUSY;
4465                 }
4466                 len = le16_to_cpu(resp->resp_len);
4467                 valid = ((u8 *)resp) + len - 1;
4468         } else {
4469                 int j;
4470
4471                 /* Check if response len is updated */
4472                 for (i = 0; i < tmo_count; i++) {
4473                         /* Abort the wait for completion if the FW health
4474                          * check has failed.
4475                          */
4476                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4477                                 return -EBUSY;
4478                         len = le16_to_cpu(resp->resp_len);
4479                         if (len)
4480                                 break;
4481                         /* on first few passes, just barely sleep */
4482                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4483                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4484                                              HWRM_SHORT_MAX_TIMEOUT);
4485                         else
4486                                 usleep_range(HWRM_MIN_TIMEOUT,
4487                                              HWRM_MAX_TIMEOUT);
4488                 }
4489
4490                 if (i >= tmo_count) {
4491                         if (!silent)
4492                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4493                                            HWRM_TOTAL_TIMEOUT(i),
4494                                            le16_to_cpu(req->req_type),
4495                                            le16_to_cpu(req->seq_id), len);
4496                         return -EBUSY;
4497                 }
4498
4499                 /* Last byte of resp contains valid bit */
4500                 valid = ((u8 *)resp) + len - 1;
4501                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4502                         /* make sure we read from updated DMA memory */
4503                         dma_rmb();
4504                         if (*valid)
4505                                 break;
4506                         usleep_range(1, 5);
4507                 }
4508
4509                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4510                         if (!silent)
4511                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4512                                            HWRM_TOTAL_TIMEOUT(i),
4513                                            le16_to_cpu(req->req_type),
4514                                            le16_to_cpu(req->seq_id), len,
4515                                            *valid);
4516                         return -EBUSY;
4517                 }
4518         }
4519
4520         /* Zero valid bit for compatibility.  Valid bit in an older spec
4521          * may become a new field in a newer spec.  We must make sure that
4522          * a new field not implemented by old spec will read zero.
4523          */
4524         *valid = 0;
4525         rc = le16_to_cpu(resp->error_code);
4526         if (rc && !silent)
4527                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4528                            le16_to_cpu(resp->req_type),
4529                            le16_to_cpu(resp->seq_id), rc);
4530         return bnxt_hwrm_to_stderr(rc);
4531 }
4532
4533 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4534 {
4535         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4536 }
4537
4538 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4539                               int timeout)
4540 {
4541         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4542 }
4543
4544 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4545 {
4546         int rc;
4547
4548         mutex_lock(&bp->hwrm_cmd_lock);
4549         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4550         mutex_unlock(&bp->hwrm_cmd_lock);
4551         return rc;
4552 }
4553
4554 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4555                              int timeout)
4556 {
4557         int rc;
4558
4559         mutex_lock(&bp->hwrm_cmd_lock);
4560         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4561         mutex_unlock(&bp->hwrm_cmd_lock);
4562         return rc;
4563 }
4564
4565 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4566                             bool async_only)
4567 {
4568         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4569         struct hwrm_func_drv_rgtr_input req = {0};
4570         DECLARE_BITMAP(async_events_bmap, 256);
4571         u32 *events = (u32 *)async_events_bmap;
4572         u32 flags;
4573         int rc, i;
4574
4575         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4576
4577         req.enables =
4578                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4579                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4580                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4581
4582         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4583         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4584         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4585                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4586         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4587                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4588                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4589         req.flags = cpu_to_le32(flags);
4590         req.ver_maj_8b = DRV_VER_MAJ;
4591         req.ver_min_8b = DRV_VER_MIN;
4592         req.ver_upd_8b = DRV_VER_UPD;
4593         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4594         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4595         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4596
4597         if (BNXT_PF(bp)) {
4598                 u32 data[8];
4599                 int i;
4600
4601                 memset(data, 0, sizeof(data));
4602                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4603                         u16 cmd = bnxt_vf_req_snif[i];
4604                         unsigned int bit, idx;
4605
4606                         idx = cmd / 32;
4607                         bit = cmd % 32;
4608                         data[idx] |= 1 << bit;
4609                 }
4610
4611                 for (i = 0; i < 8; i++)
4612                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4613
4614                 req.enables |=
4615                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4616         }
4617
4618         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4619                 req.flags |= cpu_to_le32(
4620                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4621
4622         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4623         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4624                 u16 event_id = bnxt_async_events_arr[i];
4625
4626                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4627                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4628                         continue;
4629                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4630         }
4631         if (bmap && bmap_size) {
4632                 for (i = 0; i < bmap_size; i++) {
4633                         if (test_bit(i, bmap))
4634                                 __set_bit(i, async_events_bmap);
4635                 }
4636         }
4637         for (i = 0; i < 8; i++)
4638                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4639
4640         if (async_only)
4641                 req.enables =
4642                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4643
4644         mutex_lock(&bp->hwrm_cmd_lock);
4645         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4646         if (!rc) {
4647                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4648                 if (resp->flags &
4649                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4650                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4651         }
4652         mutex_unlock(&bp->hwrm_cmd_lock);
4653         return rc;
4654 }
4655
4656 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4657 {
4658         struct hwrm_func_drv_unrgtr_input req = {0};
4659
4660         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4661                 return 0;
4662
4663         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4664         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4665 }
4666
4667 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4668 {
4669         u32 rc = 0;
4670         struct hwrm_tunnel_dst_port_free_input req = {0};
4671
4672         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4673         req.tunnel_type = tunnel_type;
4674
4675         switch (tunnel_type) {
4676         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4677                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4678                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4679                 break;
4680         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4681                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4682                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4683                 break;
4684         default:
4685                 break;
4686         }
4687
4688         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4689         if (rc)
4690                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4691                            rc);
4692         return rc;
4693 }
4694
4695 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4696                                            u8 tunnel_type)
4697 {
4698         u32 rc = 0;
4699         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4700         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4701
4702         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4703
4704         req.tunnel_type = tunnel_type;
4705         req.tunnel_dst_port_val = port;
4706
4707         mutex_lock(&bp->hwrm_cmd_lock);
4708         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4709         if (rc) {
4710                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4711                            rc);
4712                 goto err_out;
4713         }
4714
4715         switch (tunnel_type) {
4716         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4717                 bp->vxlan_fw_dst_port_id =
4718                         le16_to_cpu(resp->tunnel_dst_port_id);
4719                 break;
4720         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4721                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4722                 break;
4723         default:
4724                 break;
4725         }
4726
4727 err_out:
4728         mutex_unlock(&bp->hwrm_cmd_lock);
4729         return rc;
4730 }
4731
4732 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4733 {
4734         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4735         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4736
4737         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4738         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4739
4740         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4741         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4742         req.mask = cpu_to_le32(vnic->rx_mask);
4743         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4744 }
4745
4746 #ifdef CONFIG_RFS_ACCEL
4747 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4748                                             struct bnxt_ntuple_filter *fltr)
4749 {
4750         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4751
4752         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4753         req.ntuple_filter_id = fltr->filter_id;
4754         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4755 }
4756
4757 #define BNXT_NTP_FLTR_FLAGS                                     \
4758         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4759          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4760          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4761          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4762          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4763          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4764          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4765          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4766          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4767          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4768          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4769          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4770          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4771          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4772
4773 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4774                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4775
4776 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4777                                              struct bnxt_ntuple_filter *fltr)
4778 {
4779         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4780         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4781         struct flow_keys *keys = &fltr->fkeys;
4782         struct bnxt_vnic_info *vnic;
4783         u32 flags = 0;
4784         int rc = 0;
4785
4786         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4787         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4788
4789         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4790                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4791                 req.dst_id = cpu_to_le16(fltr->rxq);
4792         } else {
4793                 vnic = &bp->vnic_info[fltr->rxq + 1];
4794                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4795         }
4796         req.flags = cpu_to_le32(flags);
4797         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4798
4799         req.ethertype = htons(ETH_P_IP);
4800         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4801         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4802         req.ip_protocol = keys->basic.ip_proto;
4803
4804         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4805                 int i;
4806
4807                 req.ethertype = htons(ETH_P_IPV6);
4808                 req.ip_addr_type =
4809                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4810                 *(struct in6_addr *)&req.src_ipaddr[0] =
4811                         keys->addrs.v6addrs.src;
4812                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4813                         keys->addrs.v6addrs.dst;
4814                 for (i = 0; i < 4; i++) {
4815                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4816                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4817                 }
4818         } else {
4819                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4820                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4821                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4822                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4823         }
4824         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4825                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4826                 req.tunnel_type =
4827                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4828         }
4829
4830         req.src_port = keys->ports.src;
4831         req.src_port_mask = cpu_to_be16(0xffff);
4832         req.dst_port = keys->ports.dst;
4833         req.dst_port_mask = cpu_to_be16(0xffff);
4834
4835         mutex_lock(&bp->hwrm_cmd_lock);
4836         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4837         if (!rc) {
4838                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4839                 fltr->filter_id = resp->ntuple_filter_id;
4840         }
4841         mutex_unlock(&bp->hwrm_cmd_lock);
4842         return rc;
4843 }
4844 #endif
4845
4846 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4847                                      u8 *mac_addr)
4848 {
4849         u32 rc = 0;
4850         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4851         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4852
4853         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4854         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4855         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4856                 req.flags |=
4857                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4858         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4859         req.enables =
4860                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4861                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4862                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4863         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4864         req.l2_addr_mask[0] = 0xff;
4865         req.l2_addr_mask[1] = 0xff;
4866         req.l2_addr_mask[2] = 0xff;
4867         req.l2_addr_mask[3] = 0xff;
4868         req.l2_addr_mask[4] = 0xff;
4869         req.l2_addr_mask[5] = 0xff;
4870
4871         mutex_lock(&bp->hwrm_cmd_lock);
4872         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4873         if (!rc)
4874                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4875                                                         resp->l2_filter_id;
4876         mutex_unlock(&bp->hwrm_cmd_lock);
4877         return rc;
4878 }
4879
4880 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4881 {
4882         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4883         int rc = 0;
4884
4885         /* Any associated ntuple filters will also be cleared by firmware. */
4886         mutex_lock(&bp->hwrm_cmd_lock);
4887         for (i = 0; i < num_of_vnics; i++) {
4888                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4889
4890                 for (j = 0; j < vnic->uc_filter_count; j++) {
4891                         struct hwrm_cfa_l2_filter_free_input req = {0};
4892
4893                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4894                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4895
4896                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4897
4898                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4899                                                 HWRM_CMD_TIMEOUT);
4900                 }
4901                 vnic->uc_filter_count = 0;
4902         }
4903         mutex_unlock(&bp->hwrm_cmd_lock);
4904
4905         return rc;
4906 }
4907
4908 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4909 {
4910         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4911         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4912         struct hwrm_vnic_tpa_cfg_input req = {0};
4913
4914         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4915                 return 0;
4916
4917         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4918
4919         if (tpa_flags) {
4920                 u16 mss = bp->dev->mtu - 40;
4921                 u32 nsegs, n, segs = 0, flags;
4922
4923                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4924                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4925                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4926                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4927                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4928                 if (tpa_flags & BNXT_FLAG_GRO)
4929                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4930
4931                 req.flags = cpu_to_le32(flags);
4932
4933                 req.enables =
4934                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4935                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4936                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4937
4938                 /* Number of segs are log2 units, and first packet is not
4939                  * included as part of this units.
4940                  */
4941                 if (mss <= BNXT_RX_PAGE_SIZE) {
4942                         n = BNXT_RX_PAGE_SIZE / mss;
4943                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4944                 } else {
4945                         n = mss / BNXT_RX_PAGE_SIZE;
4946                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4947                                 n++;
4948                         nsegs = (MAX_SKB_FRAGS - n) / n;
4949                 }
4950
4951                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4952                         segs = MAX_TPA_SEGS_P5;
4953                         max_aggs = bp->max_tpa;
4954                 } else {
4955                         segs = ilog2(nsegs);
4956                 }
4957                 req.max_agg_segs = cpu_to_le16(segs);
4958                 req.max_aggs = cpu_to_le16(max_aggs);
4959
4960                 req.min_agg_len = cpu_to_le32(512);
4961         }
4962         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4963
4964         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4965 }
4966
4967 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4968 {
4969         struct bnxt_ring_grp_info *grp_info;
4970
4971         grp_info = &bp->grp_info[ring->grp_idx];
4972         return grp_info->cp_fw_ring_id;
4973 }
4974
4975 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4976 {
4977         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4978                 struct bnxt_napi *bnapi = rxr->bnapi;
4979                 struct bnxt_cp_ring_info *cpr;
4980
4981                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4982                 return cpr->cp_ring_struct.fw_ring_id;
4983         } else {
4984                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4985         }
4986 }
4987
4988 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4989 {
4990         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4991                 struct bnxt_napi *bnapi = txr->bnapi;
4992                 struct bnxt_cp_ring_info *cpr;
4993
4994                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4995                 return cpr->cp_ring_struct.fw_ring_id;
4996         } else {
4997                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4998         }
4999 }
5000
5001 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5002 {
5003         int entries;
5004
5005         if (bp->flags & BNXT_FLAG_CHIP_P5)
5006                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5007         else
5008                 entries = HW_HASH_INDEX_SIZE;
5009
5010         bp->rss_indir_tbl_entries = entries;
5011         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5012                                           GFP_KERNEL);
5013         if (!bp->rss_indir_tbl)
5014                 return -ENOMEM;
5015         return 0;
5016 }
5017
5018 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5019 {
5020         u16 max_rings, max_entries, pad, i;
5021
5022         if (!bp->rx_nr_rings)
5023                 return;
5024
5025         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5026                 max_rings = bp->rx_nr_rings - 1;
5027         else
5028                 max_rings = bp->rx_nr_rings;
5029
5030         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5031
5032         for (i = 0; i < max_entries; i++)
5033                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5034
5035         pad = bp->rss_indir_tbl_entries - max_entries;
5036         if (pad)
5037                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5038 }
5039
5040 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5041 {
5042         u16 i, tbl_size, max_ring = 0;
5043
5044         if (!bp->rss_indir_tbl)
5045                 return 0;
5046
5047         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5048         for (i = 0; i < tbl_size; i++)
5049                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5050         return max_ring;
5051 }
5052
5053 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5054 {
5055         if (bp->flags & BNXT_FLAG_CHIP_P5)
5056                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5057         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5058                 return 2;
5059         return 1;
5060 }
5061
5062 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5063 {
5064         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5065         u16 i, j;
5066
5067         /* Fill the RSS indirection table with ring group ids */
5068         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5069                 if (!no_rss)
5070                         j = bp->rss_indir_tbl[i];
5071                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5072         }
5073 }
5074
5075 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5076                                       struct bnxt_vnic_info *vnic)
5077 {
5078         __le16 *ring_tbl = vnic->rss_table;
5079         struct bnxt_rx_ring_info *rxr;
5080         u16 tbl_size, i;
5081
5082         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5083
5084         for (i = 0; i < tbl_size; i++) {
5085                 u16 ring_id, j;
5086
5087                 j = bp->rss_indir_tbl[i];
5088                 rxr = &bp->rx_ring[j];
5089
5090                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5091                 *ring_tbl++ = cpu_to_le16(ring_id);
5092                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5093                 *ring_tbl++ = cpu_to_le16(ring_id);
5094         }
5095 }
5096
5097 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5098 {
5099         if (bp->flags & BNXT_FLAG_CHIP_P5)
5100                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5101         else
5102                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5103 }
5104
5105 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5106 {
5107         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5108         struct hwrm_vnic_rss_cfg_input req = {0};
5109
5110         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5111             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5112                 return 0;
5113
5114         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5115         if (set_rss) {
5116                 bnxt_fill_hw_rss_tbl(bp, vnic);
5117                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5118                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5119                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5120                 req.hash_key_tbl_addr =
5121                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5122         }
5123         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5124         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5125 }
5126
5127 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5128 {
5129         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5130         struct hwrm_vnic_rss_cfg_input req = {0};
5131         dma_addr_t ring_tbl_map;
5132         u32 i, nr_ctxs;
5133
5134         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5135         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5136         if (!set_rss) {
5137                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5138                 return 0;
5139         }
5140         bnxt_fill_hw_rss_tbl(bp, vnic);
5141         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5142         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5143         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5144         ring_tbl_map = vnic->rss_table_dma_addr;
5145         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5146         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5147                 int rc;
5148
5149                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5150                 req.ring_table_pair_index = i;
5151                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5152                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5153                 if (rc)
5154                         return rc;
5155         }
5156         return 0;
5157 }
5158
5159 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5160 {
5161         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5162         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5163
5164         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5165         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5166                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5167                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5168         req.enables =
5169                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5170                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5171         /* thresholds not implemented in firmware yet */
5172         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5173         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5174         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5175         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5176 }
5177
5178 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5179                                         u16 ctx_idx)
5180 {
5181         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5182
5183         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5184         req.rss_cos_lb_ctx_id =
5185                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5186
5187         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5188         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5189 }
5190
5191 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5192 {
5193         int i, j;
5194
5195         for (i = 0; i < bp->nr_vnics; i++) {
5196                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5197
5198                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5199                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5200                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5201                 }
5202         }
5203         bp->rsscos_nr_ctxs = 0;
5204 }
5205
5206 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5207 {
5208         int rc;
5209         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5210         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5211                                                 bp->hwrm_cmd_resp_addr;
5212
5213         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5214                                -1);
5215
5216         mutex_lock(&bp->hwrm_cmd_lock);
5217         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5218         if (!rc)
5219                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5220                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5221         mutex_unlock(&bp->hwrm_cmd_lock);
5222
5223         return rc;
5224 }
5225
5226 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5227 {
5228         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5229                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5230         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5231 }
5232
5233 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5234 {
5235         unsigned int ring = 0, grp_idx;
5236         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5237         struct hwrm_vnic_cfg_input req = {0};
5238         u16 def_vlan = 0;
5239
5240         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5241
5242         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5243                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5244
5245                 req.default_rx_ring_id =
5246                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5247                 req.default_cmpl_ring_id =
5248                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5249                 req.enables =
5250                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5251                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5252                 goto vnic_mru;
5253         }
5254         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5255         /* Only RSS support for now TBD: COS & LB */
5256         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5257                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5258                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5259                                            VNIC_CFG_REQ_ENABLES_MRU);
5260         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5261                 req.rss_rule =
5262                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5263                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5264                                            VNIC_CFG_REQ_ENABLES_MRU);
5265                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5266         } else {
5267                 req.rss_rule = cpu_to_le16(0xffff);
5268         }
5269
5270         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5271             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5272                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5273                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5274         } else {
5275                 req.cos_rule = cpu_to_le16(0xffff);
5276         }
5277
5278         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5279                 ring = 0;
5280         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5281                 ring = vnic_id - 1;
5282         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5283                 ring = bp->rx_nr_rings - 1;
5284
5285         grp_idx = bp->rx_ring[ring].bnapi->index;
5286         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5287         req.lb_rule = cpu_to_le16(0xffff);
5288 vnic_mru:
5289         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5290
5291         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5292 #ifdef CONFIG_BNXT_SRIOV
5293         if (BNXT_VF(bp))
5294                 def_vlan = bp->vf.vlan;
5295 #endif
5296         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5297                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5298         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5299                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5300
5301         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5302 }
5303
5304 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5305 {
5306         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5307                 struct hwrm_vnic_free_input req = {0};
5308
5309                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5310                 req.vnic_id =
5311                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5312
5313                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5314                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5315         }
5316 }
5317
5318 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5319 {
5320         u16 i;
5321
5322         for (i = 0; i < bp->nr_vnics; i++)
5323                 bnxt_hwrm_vnic_free_one(bp, i);
5324 }
5325
5326 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5327                                 unsigned int start_rx_ring_idx,
5328                                 unsigned int nr_rings)
5329 {
5330         int rc = 0;
5331         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5332         struct hwrm_vnic_alloc_input req = {0};
5333         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5334         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5335
5336         if (bp->flags & BNXT_FLAG_CHIP_P5)
5337                 goto vnic_no_ring_grps;
5338
5339         /* map ring groups to this vnic */
5340         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5341                 grp_idx = bp->rx_ring[i].bnapi->index;
5342                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5343                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5344                                    j, nr_rings);
5345                         break;
5346                 }
5347                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5348         }
5349
5350 vnic_no_ring_grps:
5351         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5352                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5353         if (vnic_id == 0)
5354                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5355
5356         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5357
5358         mutex_lock(&bp->hwrm_cmd_lock);
5359         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5360         if (!rc)
5361                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5362         mutex_unlock(&bp->hwrm_cmd_lock);
5363         return rc;
5364 }
5365
5366 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5367 {
5368         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5369         struct hwrm_vnic_qcaps_input req = {0};
5370         int rc;
5371
5372         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5373         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5374         if (bp->hwrm_spec_code < 0x10600)
5375                 return 0;
5376
5377         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5378         mutex_lock(&bp->hwrm_cmd_lock);
5379         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5380         if (!rc) {
5381                 u32 flags = le32_to_cpu(resp->flags);
5382
5383                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5384                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5385                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5386                 if (flags &
5387                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5388                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5389
5390                 /* Older P5 fw before EXT_HW_STATS support did not set
5391                  * VLAN_STRIP_CAP properly.
5392                  */
5393                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5394                     (BNXT_CHIP_P5_THOR(bp) &&
5395                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5396                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5397                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5398                 if (bp->max_tpa_v2) {
5399                         if (BNXT_CHIP_P5_THOR(bp))
5400                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5401                         else
5402                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5403                 }
5404         }
5405         mutex_unlock(&bp->hwrm_cmd_lock);
5406         return rc;
5407 }
5408
5409 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5410 {
5411         u16 i;
5412         u32 rc = 0;
5413
5414         if (bp->flags & BNXT_FLAG_CHIP_P5)
5415                 return 0;
5416
5417         mutex_lock(&bp->hwrm_cmd_lock);
5418         for (i = 0; i < bp->rx_nr_rings; i++) {
5419                 struct hwrm_ring_grp_alloc_input req = {0};
5420                 struct hwrm_ring_grp_alloc_output *resp =
5421                                         bp->hwrm_cmd_resp_addr;
5422                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5423
5424                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5425
5426                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5427                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5428                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5429                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5430
5431                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5432                                         HWRM_CMD_TIMEOUT);
5433                 if (rc)
5434                         break;
5435
5436                 bp->grp_info[grp_idx].fw_grp_id =
5437                         le32_to_cpu(resp->ring_group_id);
5438         }
5439         mutex_unlock(&bp->hwrm_cmd_lock);
5440         return rc;
5441 }
5442
5443 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5444 {
5445         u16 i;
5446         struct hwrm_ring_grp_free_input req = {0};
5447
5448         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5449                 return;
5450
5451         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5452
5453         mutex_lock(&bp->hwrm_cmd_lock);
5454         for (i = 0; i < bp->cp_nr_rings; i++) {
5455                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5456                         continue;
5457                 req.ring_group_id =
5458                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5459
5460                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5461                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5462         }
5463         mutex_unlock(&bp->hwrm_cmd_lock);
5464 }
5465
5466 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5467                                     struct bnxt_ring_struct *ring,
5468                                     u32 ring_type, u32 map_index)
5469 {
5470         int rc = 0, err = 0;
5471         struct hwrm_ring_alloc_input req = {0};
5472         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5473         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5474         struct bnxt_ring_grp_info *grp_info;
5475         u16 ring_id;
5476
5477         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5478
5479         req.enables = 0;
5480         if (rmem->nr_pages > 1) {
5481                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5482                 /* Page size is in log2 units */
5483                 req.page_size = BNXT_PAGE_SHIFT;
5484                 req.page_tbl_depth = 1;
5485         } else {
5486                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5487         }
5488         req.fbo = 0;
5489         /* Association of ring index with doorbell index and MSIX number */
5490         req.logical_id = cpu_to_le16(map_index);
5491
5492         switch (ring_type) {
5493         case HWRM_RING_ALLOC_TX: {
5494                 struct bnxt_tx_ring_info *txr;
5495
5496                 txr = container_of(ring, struct bnxt_tx_ring_info,
5497                                    tx_ring_struct);
5498                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5499                 /* Association of transmit ring with completion ring */
5500                 grp_info = &bp->grp_info[ring->grp_idx];
5501                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5502                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5503                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5504                 req.queue_id = cpu_to_le16(ring->queue_id);
5505                 break;
5506         }
5507         case HWRM_RING_ALLOC_RX:
5508                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5509                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5510                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5511                         u16 flags = 0;
5512
5513                         /* Association of rx ring with stats context */
5514                         grp_info = &bp->grp_info[ring->grp_idx];
5515                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5516                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5517                         req.enables |= cpu_to_le32(
5518                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5519                         if (NET_IP_ALIGN == 2)
5520                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5521                         req.flags = cpu_to_le16(flags);
5522                 }
5523                 break;
5524         case HWRM_RING_ALLOC_AGG:
5525                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5526                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5527                         /* Association of agg ring with rx ring */
5528                         grp_info = &bp->grp_info[ring->grp_idx];
5529                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5530                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5531                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5532                         req.enables |= cpu_to_le32(
5533                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5534                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5535                 } else {
5536                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5537                 }
5538                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5539                 break;
5540         case HWRM_RING_ALLOC_CMPL:
5541                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5542                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5543                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5544                         /* Association of cp ring with nq */
5545                         grp_info = &bp->grp_info[map_index];
5546                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5547                         req.cq_handle = cpu_to_le64(ring->handle);
5548                         req.enables |= cpu_to_le32(
5549                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5550                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5551                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5552                 }
5553                 break;
5554         case HWRM_RING_ALLOC_NQ:
5555                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5556                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5557                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5558                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5559                 break;
5560         default:
5561                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5562                            ring_type);
5563                 return -1;
5564         }
5565
5566         mutex_lock(&bp->hwrm_cmd_lock);
5567         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5568         err = le16_to_cpu(resp->error_code);
5569         ring_id = le16_to_cpu(resp->ring_id);
5570         mutex_unlock(&bp->hwrm_cmd_lock);
5571
5572         if (rc || err) {
5573                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5574                            ring_type, rc, err);
5575                 return -EIO;
5576         }
5577         ring->fw_ring_id = ring_id;
5578         return rc;
5579 }
5580
5581 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5582 {
5583         int rc;
5584
5585         if (BNXT_PF(bp)) {
5586                 struct hwrm_func_cfg_input req = {0};
5587
5588                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5589                 req.fid = cpu_to_le16(0xffff);
5590                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5591                 req.async_event_cr = cpu_to_le16(idx);
5592                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5593         } else {
5594                 struct hwrm_func_vf_cfg_input req = {0};
5595
5596                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5597                 req.enables =
5598                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5599                 req.async_event_cr = cpu_to_le16(idx);
5600                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5601         }
5602         return rc;
5603 }
5604
5605 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5606                         u32 map_idx, u32 xid)
5607 {
5608         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5609                 if (BNXT_PF(bp))
5610                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5611                 else
5612                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5613                 switch (ring_type) {
5614                 case HWRM_RING_ALLOC_TX:
5615                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5616                         break;
5617                 case HWRM_RING_ALLOC_RX:
5618                 case HWRM_RING_ALLOC_AGG:
5619                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5620                         break;
5621                 case HWRM_RING_ALLOC_CMPL:
5622                         db->db_key64 = DBR_PATH_L2;
5623                         break;
5624                 case HWRM_RING_ALLOC_NQ:
5625                         db->db_key64 = DBR_PATH_L2;
5626                         break;
5627                 }
5628                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5629         } else {
5630                 db->doorbell = bp->bar1 + map_idx * 0x80;
5631                 switch (ring_type) {
5632                 case HWRM_RING_ALLOC_TX:
5633                         db->db_key32 = DB_KEY_TX;
5634                         break;
5635                 case HWRM_RING_ALLOC_RX:
5636                 case HWRM_RING_ALLOC_AGG:
5637                         db->db_key32 = DB_KEY_RX;
5638                         break;
5639                 case HWRM_RING_ALLOC_CMPL:
5640                         db->db_key32 = DB_KEY_CP;
5641                         break;
5642                 }
5643         }
5644 }
5645
5646 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5647 {
5648         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5649         int i, rc = 0;
5650         u32 type;
5651
5652         if (bp->flags & BNXT_FLAG_CHIP_P5)
5653                 type = HWRM_RING_ALLOC_NQ;
5654         else
5655                 type = HWRM_RING_ALLOC_CMPL;
5656         for (i = 0; i < bp->cp_nr_rings; i++) {
5657                 struct bnxt_napi *bnapi = bp->bnapi[i];
5658                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5659                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5660                 u32 map_idx = ring->map_idx;
5661                 unsigned int vector;
5662
5663                 vector = bp->irq_tbl[map_idx].vector;
5664                 disable_irq_nosync(vector);
5665                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5666                 if (rc) {
5667                         enable_irq(vector);
5668                         goto err_out;
5669                 }
5670                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5671                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5672                 enable_irq(vector);
5673                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5674
5675                 if (!i) {
5676                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5677                         if (rc)
5678                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5679                 }
5680         }
5681
5682         type = HWRM_RING_ALLOC_TX;
5683         for (i = 0; i < bp->tx_nr_rings; i++) {
5684                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5685                 struct bnxt_ring_struct *ring;
5686                 u32 map_idx;
5687
5688                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5689                         struct bnxt_napi *bnapi = txr->bnapi;
5690                         struct bnxt_cp_ring_info *cpr, *cpr2;
5691                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5692
5693                         cpr = &bnapi->cp_ring;
5694                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5695                         ring = &cpr2->cp_ring_struct;
5696                         ring->handle = BNXT_TX_HDL;
5697                         map_idx = bnapi->index;
5698                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5699                         if (rc)
5700                                 goto err_out;
5701                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5702                                     ring->fw_ring_id);
5703                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5704                 }
5705                 ring = &txr->tx_ring_struct;
5706                 map_idx = i;
5707                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5708                 if (rc)
5709                         goto err_out;
5710                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5711         }
5712
5713         type = HWRM_RING_ALLOC_RX;
5714         for (i = 0; i < bp->rx_nr_rings; i++) {
5715                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5716                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5717                 struct bnxt_napi *bnapi = rxr->bnapi;
5718                 u32 map_idx = bnapi->index;
5719
5720                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5721                 if (rc)
5722                         goto err_out;
5723                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5724                 /* If we have agg rings, post agg buffers first. */
5725                 if (!agg_rings)
5726                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5727                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5728                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5729                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5730                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5731                         struct bnxt_cp_ring_info *cpr2;
5732
5733                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5734                         ring = &cpr2->cp_ring_struct;
5735                         ring->handle = BNXT_RX_HDL;
5736                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5737                         if (rc)
5738                                 goto err_out;
5739                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5740                                     ring->fw_ring_id);
5741                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5742                 }
5743         }
5744
5745         if (agg_rings) {
5746                 type = HWRM_RING_ALLOC_AGG;
5747                 for (i = 0; i < bp->rx_nr_rings; i++) {
5748                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5749                         struct bnxt_ring_struct *ring =
5750                                                 &rxr->rx_agg_ring_struct;
5751                         u32 grp_idx = ring->grp_idx;
5752                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5753
5754                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5755                         if (rc)
5756                                 goto err_out;
5757
5758                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5759                                     ring->fw_ring_id);
5760                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5761                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5762                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5763                 }
5764         }
5765 err_out:
5766         return rc;
5767 }
5768
5769 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5770                                    struct bnxt_ring_struct *ring,
5771                                    u32 ring_type, int cmpl_ring_id)
5772 {
5773         int rc;
5774         struct hwrm_ring_free_input req = {0};
5775         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5776         u16 error_code;
5777
5778         if (BNXT_NO_FW_ACCESS(bp))
5779                 return 0;
5780
5781         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5782         req.ring_type = ring_type;
5783         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5784
5785         mutex_lock(&bp->hwrm_cmd_lock);
5786         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5787         error_code = le16_to_cpu(resp->error_code);
5788         mutex_unlock(&bp->hwrm_cmd_lock);
5789
5790         if (rc || error_code) {
5791                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5792                            ring_type, rc, error_code);
5793                 return -EIO;
5794         }
5795         return 0;
5796 }
5797
5798 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5799 {
5800         u32 type;
5801         int i;
5802
5803         if (!bp->bnapi)
5804                 return;
5805
5806         for (i = 0; i < bp->tx_nr_rings; i++) {
5807                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5808                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5809
5810                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5811                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5812
5813                         hwrm_ring_free_send_msg(bp, ring,
5814                                                 RING_FREE_REQ_RING_TYPE_TX,
5815                                                 close_path ? cmpl_ring_id :
5816                                                 INVALID_HW_RING_ID);
5817                         ring->fw_ring_id = INVALID_HW_RING_ID;
5818                 }
5819         }
5820
5821         for (i = 0; i < bp->rx_nr_rings; i++) {
5822                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5823                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5824                 u32 grp_idx = rxr->bnapi->index;
5825
5826                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5827                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5828
5829                         hwrm_ring_free_send_msg(bp, ring,
5830                                                 RING_FREE_REQ_RING_TYPE_RX,
5831                                                 close_path ? cmpl_ring_id :
5832                                                 INVALID_HW_RING_ID);
5833                         ring->fw_ring_id = INVALID_HW_RING_ID;
5834                         bp->grp_info[grp_idx].rx_fw_ring_id =
5835                                 INVALID_HW_RING_ID;
5836                 }
5837         }
5838
5839         if (bp->flags & BNXT_FLAG_CHIP_P5)
5840                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5841         else
5842                 type = RING_FREE_REQ_RING_TYPE_RX;
5843         for (i = 0; i < bp->rx_nr_rings; i++) {
5844                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5845                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5846                 u32 grp_idx = rxr->bnapi->index;
5847
5848                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5849                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5850
5851                         hwrm_ring_free_send_msg(bp, ring, type,
5852                                                 close_path ? cmpl_ring_id :
5853                                                 INVALID_HW_RING_ID);
5854                         ring->fw_ring_id = INVALID_HW_RING_ID;
5855                         bp->grp_info[grp_idx].agg_fw_ring_id =
5856                                 INVALID_HW_RING_ID;
5857                 }
5858         }
5859
5860         /* The completion rings are about to be freed.  After that the
5861          * IRQ doorbell will not work anymore.  So we need to disable
5862          * IRQ here.
5863          */
5864         bnxt_disable_int_sync(bp);
5865
5866         if (bp->flags & BNXT_FLAG_CHIP_P5)
5867                 type = RING_FREE_REQ_RING_TYPE_NQ;
5868         else
5869                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5870         for (i = 0; i < bp->cp_nr_rings; i++) {
5871                 struct bnxt_napi *bnapi = bp->bnapi[i];
5872                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5873                 struct bnxt_ring_struct *ring;
5874                 int j;
5875
5876                 for (j = 0; j < 2; j++) {
5877                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5878
5879                         if (cpr2) {
5880                                 ring = &cpr2->cp_ring_struct;
5881                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5882                                         continue;
5883                                 hwrm_ring_free_send_msg(bp, ring,
5884                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5885                                         INVALID_HW_RING_ID);
5886                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5887                         }
5888                 }
5889                 ring = &cpr->cp_ring_struct;
5890                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5891                         hwrm_ring_free_send_msg(bp, ring, type,
5892                                                 INVALID_HW_RING_ID);
5893                         ring->fw_ring_id = INVALID_HW_RING_ID;
5894                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5895                 }
5896         }
5897 }
5898
5899 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5900                            bool shared);
5901
5902 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5903 {
5904         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5905         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5906         struct hwrm_func_qcfg_input req = {0};
5907         int rc;
5908
5909         if (bp->hwrm_spec_code < 0x10601)
5910                 return 0;
5911
5912         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5913         req.fid = cpu_to_le16(0xffff);
5914         mutex_lock(&bp->hwrm_cmd_lock);
5915         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5916         if (rc) {
5917                 mutex_unlock(&bp->hwrm_cmd_lock);
5918                 return rc;
5919         }
5920
5921         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5922         if (BNXT_NEW_RM(bp)) {
5923                 u16 cp, stats;
5924
5925                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5926                 hw_resc->resv_hw_ring_grps =
5927                         le32_to_cpu(resp->alloc_hw_ring_grps);
5928                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5929                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5930                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5931                 hw_resc->resv_irqs = cp;
5932                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5933                         int rx = hw_resc->resv_rx_rings;
5934                         int tx = hw_resc->resv_tx_rings;
5935
5936                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5937                                 rx >>= 1;
5938                         if (cp < (rx + tx)) {
5939                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5940                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5941                                         rx <<= 1;
5942                                 hw_resc->resv_rx_rings = rx;
5943                                 hw_resc->resv_tx_rings = tx;
5944                         }
5945                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5946                         hw_resc->resv_hw_ring_grps = rx;
5947                 }
5948                 hw_resc->resv_cp_rings = cp;
5949                 hw_resc->resv_stat_ctxs = stats;
5950         }
5951         mutex_unlock(&bp->hwrm_cmd_lock);
5952         return 0;
5953 }
5954
5955 /* Caller must hold bp->hwrm_cmd_lock */
5956 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5957 {
5958         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5959         struct hwrm_func_qcfg_input req = {0};
5960         int rc;
5961
5962         if (bp->hwrm_spec_code < 0x10601)
5963                 return 0;
5964
5965         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5966         req.fid = cpu_to_le16(fid);
5967         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5968         if (!rc)
5969                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5970
5971         return rc;
5972 }
5973
5974 static bool bnxt_rfs_supported(struct bnxt *bp);
5975
5976 static void
5977 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5978                              int tx_rings, int rx_rings, int ring_grps,
5979                              int cp_rings, int stats, int vnics)
5980 {
5981         u32 enables = 0;
5982
5983         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5984         req->fid = cpu_to_le16(0xffff);
5985         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5986         req->num_tx_rings = cpu_to_le16(tx_rings);
5987         if (BNXT_NEW_RM(bp)) {
5988                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5989                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5990                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5991                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5992                         enables |= tx_rings + ring_grps ?
5993                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5994                         enables |= rx_rings ?
5995                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5996                 } else {
5997                         enables |= cp_rings ?
5998                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5999                         enables |= ring_grps ?
6000                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6001                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6002                 }
6003                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6004
6005                 req->num_rx_rings = cpu_to_le16(rx_rings);
6006                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6007                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6008                         req->num_msix = cpu_to_le16(cp_rings);
6009                         req->num_rsscos_ctxs =
6010                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6011                 } else {
6012                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6013                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6014                         req->num_rsscos_ctxs = cpu_to_le16(1);
6015                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6016                             bnxt_rfs_supported(bp))
6017                                 req->num_rsscos_ctxs =
6018                                         cpu_to_le16(ring_grps + 1);
6019                 }
6020                 req->num_stat_ctxs = cpu_to_le16(stats);
6021                 req->num_vnics = cpu_to_le16(vnics);
6022         }
6023         req->enables = cpu_to_le32(enables);
6024 }
6025
6026 static void
6027 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6028                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
6029                              int rx_rings, int ring_grps, int cp_rings,
6030                              int stats, int vnics)
6031 {
6032         u32 enables = 0;
6033
6034         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6035         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6036         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6037                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6038         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6039         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6040                 enables |= tx_rings + ring_grps ?
6041                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6042         } else {
6043                 enables |= cp_rings ?
6044                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6045                 enables |= ring_grps ?
6046                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6047         }
6048         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6049         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6050
6051         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6052         req->num_tx_rings = cpu_to_le16(tx_rings);
6053         req->num_rx_rings = cpu_to_le16(rx_rings);
6054         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6055                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6056                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6057         } else {
6058                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6059                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6060                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6061         }
6062         req->num_stat_ctxs = cpu_to_le16(stats);
6063         req->num_vnics = cpu_to_le16(vnics);
6064
6065         req->enables = cpu_to_le32(enables);
6066 }
6067
6068 static int
6069 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6070                            int ring_grps, int cp_rings, int stats, int vnics)
6071 {
6072         struct hwrm_func_cfg_input req = {0};
6073         int rc;
6074
6075         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6076                                      cp_rings, stats, vnics);
6077         if (!req.enables)
6078                 return 0;
6079
6080         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6081         if (rc)
6082                 return rc;
6083
6084         if (bp->hwrm_spec_code < 0x10601)
6085                 bp->hw_resc.resv_tx_rings = tx_rings;
6086
6087         return bnxt_hwrm_get_rings(bp);
6088 }
6089
6090 static int
6091 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6092                            int ring_grps, int cp_rings, int stats, int vnics)
6093 {
6094         struct hwrm_func_vf_cfg_input req = {0};
6095         int rc;
6096
6097         if (!BNXT_NEW_RM(bp)) {
6098                 bp->hw_resc.resv_tx_rings = tx_rings;
6099                 return 0;
6100         }
6101
6102         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6103                                      cp_rings, stats, vnics);
6104         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6105         if (rc)
6106                 return rc;
6107
6108         return bnxt_hwrm_get_rings(bp);
6109 }
6110
6111 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6112                                    int cp, int stat, int vnic)
6113 {
6114         if (BNXT_PF(bp))
6115                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6116                                                   vnic);
6117         else
6118                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6119                                                   vnic);
6120 }
6121
6122 int bnxt_nq_rings_in_use(struct bnxt *bp)
6123 {
6124         int cp = bp->cp_nr_rings;
6125         int ulp_msix, ulp_base;
6126
6127         ulp_msix = bnxt_get_ulp_msix_num(bp);
6128         if (ulp_msix) {
6129                 ulp_base = bnxt_get_ulp_msix_base(bp);
6130                 cp += ulp_msix;
6131                 if ((ulp_base + ulp_msix) > cp)
6132                         cp = ulp_base + ulp_msix;
6133         }
6134         return cp;
6135 }
6136
6137 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6138 {
6139         int cp;
6140
6141         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6142                 return bnxt_nq_rings_in_use(bp);
6143
6144         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6145         return cp;
6146 }
6147
6148 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6149 {
6150         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6151         int cp = bp->cp_nr_rings;
6152
6153         if (!ulp_stat)
6154                 return cp;
6155
6156         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6157                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6158
6159         return cp + ulp_stat;
6160 }
6161
6162 /* Check if a default RSS map needs to be setup.  This function is only
6163  * used on older firmware that does not require reserving RX rings.
6164  */
6165 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6166 {
6167         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6168
6169         /* The RSS map is valid for RX rings set to resv_rx_rings */
6170         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6171                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6172                 if (!netif_is_rxfh_configured(bp->dev))
6173                         bnxt_set_dflt_rss_indir_tbl(bp);
6174         }
6175 }
6176
6177 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6178 {
6179         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6180         int cp = bnxt_cp_rings_in_use(bp);
6181         int nq = bnxt_nq_rings_in_use(bp);
6182         int rx = bp->rx_nr_rings, stat;
6183         int vnic = 1, grp = rx;
6184
6185         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6186             bp->hwrm_spec_code >= 0x10601)
6187                 return true;
6188
6189         /* Old firmware does not need RX ring reservations but we still
6190          * need to setup a default RSS map when needed.  With new firmware
6191          * we go through RX ring reservations first and then set up the
6192          * RSS map for the successfully reserved RX rings when needed.
6193          */
6194         if (!BNXT_NEW_RM(bp)) {
6195                 bnxt_check_rss_tbl_no_rmgr(bp);
6196                 return false;
6197         }
6198         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6199                 vnic = rx + 1;
6200         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6201                 rx <<= 1;
6202         stat = bnxt_get_func_stat_ctxs(bp);
6203         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6204             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6205             (hw_resc->resv_hw_ring_grps != grp &&
6206              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6207                 return true;
6208         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6209             hw_resc->resv_irqs != nq)
6210                 return true;
6211         return false;
6212 }
6213
6214 static int __bnxt_reserve_rings(struct bnxt *bp)
6215 {
6216         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6217         int cp = bnxt_nq_rings_in_use(bp);
6218         int tx = bp->tx_nr_rings;
6219         int rx = bp->rx_nr_rings;
6220         int grp, rx_rings, rc;
6221         int vnic = 1, stat;
6222         bool sh = false;
6223
6224         if (!bnxt_need_reserve_rings(bp))
6225                 return 0;
6226
6227         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6228                 sh = true;
6229         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6230                 vnic = rx + 1;
6231         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6232                 rx <<= 1;
6233         grp = bp->rx_nr_rings;
6234         stat = bnxt_get_func_stat_ctxs(bp);
6235
6236         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6237         if (rc)
6238                 return rc;
6239
6240         tx = hw_resc->resv_tx_rings;
6241         if (BNXT_NEW_RM(bp)) {
6242                 rx = hw_resc->resv_rx_rings;
6243                 cp = hw_resc->resv_irqs;
6244                 grp = hw_resc->resv_hw_ring_grps;
6245                 vnic = hw_resc->resv_vnics;
6246                 stat = hw_resc->resv_stat_ctxs;
6247         }
6248
6249         rx_rings = rx;
6250         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6251                 if (rx >= 2) {
6252                         rx_rings = rx >> 1;
6253                 } else {
6254                         if (netif_running(bp->dev))
6255                                 return -ENOMEM;
6256
6257                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6258                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6259                         bp->dev->hw_features &= ~NETIF_F_LRO;
6260                         bp->dev->features &= ~NETIF_F_LRO;
6261                         bnxt_set_ring_params(bp);
6262                 }
6263         }
6264         rx_rings = min_t(int, rx_rings, grp);
6265         cp = min_t(int, cp, bp->cp_nr_rings);
6266         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6267                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6268         cp = min_t(int, cp, stat);
6269         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6270         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6271                 rx = rx_rings << 1;
6272         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6273         bp->tx_nr_rings = tx;
6274
6275         /* If we cannot reserve all the RX rings, reset the RSS map only
6276          * if absolutely necessary
6277          */
6278         if (rx_rings != bp->rx_nr_rings) {
6279                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6280                             rx_rings, bp->rx_nr_rings);
6281                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6282                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6283                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6284                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6285                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6286                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6287                 }
6288         }
6289         bp->rx_nr_rings = rx_rings;
6290         bp->cp_nr_rings = cp;
6291
6292         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6293                 return -ENOMEM;
6294
6295         if (!netif_is_rxfh_configured(bp->dev))
6296                 bnxt_set_dflt_rss_indir_tbl(bp);
6297
6298         return rc;
6299 }
6300
6301 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6302                                     int ring_grps, int cp_rings, int stats,
6303                                     int vnics)
6304 {
6305         struct hwrm_func_vf_cfg_input req = {0};
6306         u32 flags;
6307
6308         if (!BNXT_NEW_RM(bp))
6309                 return 0;
6310
6311         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6312                                      cp_rings, stats, vnics);
6313         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6314                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6315                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6316                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6317                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6318                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6319         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6320                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6321
6322         req.flags = cpu_to_le32(flags);
6323         return hwrm_send_message_silent(bp, &req, sizeof(req),
6324                                         HWRM_CMD_TIMEOUT);
6325 }
6326
6327 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6328                                     int ring_grps, int cp_rings, int stats,
6329                                     int vnics)
6330 {
6331         struct hwrm_func_cfg_input req = {0};
6332         u32 flags;
6333
6334         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6335                                      cp_rings, stats, vnics);
6336         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6337         if (BNXT_NEW_RM(bp)) {
6338                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6339                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6340                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6341                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6342                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6343                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6344                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6345                 else
6346                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6347         }
6348
6349         req.flags = cpu_to_le32(flags);
6350         return hwrm_send_message_silent(bp, &req, sizeof(req),
6351                                         HWRM_CMD_TIMEOUT);
6352 }
6353
6354 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6355                                  int ring_grps, int cp_rings, int stats,
6356                                  int vnics)
6357 {
6358         if (bp->hwrm_spec_code < 0x10801)
6359                 return 0;
6360
6361         if (BNXT_PF(bp))
6362                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6363                                                 ring_grps, cp_rings, stats,
6364                                                 vnics);
6365
6366         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6367                                         cp_rings, stats, vnics);
6368 }
6369
6370 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6371 {
6372         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6373         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6374         struct hwrm_ring_aggint_qcaps_input req = {0};
6375         int rc;
6376
6377         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6378         coal_cap->num_cmpl_dma_aggr_max = 63;
6379         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6380         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6381         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6382         coal_cap->int_lat_tmr_min_max = 65535;
6383         coal_cap->int_lat_tmr_max_max = 65535;
6384         coal_cap->num_cmpl_aggr_int_max = 65535;
6385         coal_cap->timer_units = 80;
6386
6387         if (bp->hwrm_spec_code < 0x10902)
6388                 return;
6389
6390         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6391         mutex_lock(&bp->hwrm_cmd_lock);
6392         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6393         if (!rc) {
6394                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6395                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6396                 coal_cap->num_cmpl_dma_aggr_max =
6397                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6398                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6399                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6400                 coal_cap->cmpl_aggr_dma_tmr_max =
6401                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6402                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6403                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6404                 coal_cap->int_lat_tmr_min_max =
6405                         le16_to_cpu(resp->int_lat_tmr_min_max);
6406                 coal_cap->int_lat_tmr_max_max =
6407                         le16_to_cpu(resp->int_lat_tmr_max_max);
6408                 coal_cap->num_cmpl_aggr_int_max =
6409                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6410                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6411         }
6412         mutex_unlock(&bp->hwrm_cmd_lock);
6413 }
6414
6415 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6416 {
6417         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6418
6419         return usec * 1000 / coal_cap->timer_units;
6420 }
6421
6422 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6423         struct bnxt_coal *hw_coal,
6424         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6425 {
6426         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6427         u32 cmpl_params = coal_cap->cmpl_params;
6428         u16 val, tmr, max, flags = 0;
6429
6430         max = hw_coal->bufs_per_record * 128;
6431         if (hw_coal->budget)
6432                 max = hw_coal->bufs_per_record * hw_coal->budget;
6433         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6434
6435         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6436         req->num_cmpl_aggr_int = cpu_to_le16(val);
6437
6438         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6439         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6440
6441         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6442                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6443         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6444
6445         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6446         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6447         req->int_lat_tmr_max = cpu_to_le16(tmr);
6448
6449         /* min timer set to 1/2 of interrupt timer */
6450         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6451                 val = tmr / 2;
6452                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6453                 req->int_lat_tmr_min = cpu_to_le16(val);
6454                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6455         }
6456
6457         /* buf timer set to 1/4 of interrupt timer */
6458         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6459         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6460
6461         if (cmpl_params &
6462             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6463                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6464                 val = clamp_t(u16, tmr, 1,
6465                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6466                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6467                 req->enables |=
6468                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6469         }
6470
6471         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6472                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6473         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6474             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6475                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6476         req->flags = cpu_to_le16(flags);
6477         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6478 }
6479
6480 /* Caller holds bp->hwrm_cmd_lock */
6481 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6482                                    struct bnxt_coal *hw_coal)
6483 {
6484         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6485         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6486         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6487         u32 nq_params = coal_cap->nq_params;
6488         u16 tmr;
6489
6490         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6491                 return 0;
6492
6493         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6494                                -1, -1);
6495         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6496         req.flags =
6497                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6498
6499         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6500         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6501         req.int_lat_tmr_min = cpu_to_le16(tmr);
6502         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6503         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6504 }
6505
6506 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6507 {
6508         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6509         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6510         struct bnxt_coal coal;
6511
6512         /* Tick values in micro seconds.
6513          * 1 coal_buf x bufs_per_record = 1 completion record.
6514          */
6515         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6516
6517         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6518         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6519
6520         if (!bnapi->rx_ring)
6521                 return -ENODEV;
6522
6523         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6524                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6525
6526         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6527
6528         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6529
6530         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6531                                  HWRM_CMD_TIMEOUT);
6532 }
6533
6534 int bnxt_hwrm_set_coal(struct bnxt *bp)
6535 {
6536         int i, rc = 0;
6537         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6538                                                            req_tx = {0}, *req;
6539
6540         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6541                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6542         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6543                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6544
6545         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6546         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6547
6548         mutex_lock(&bp->hwrm_cmd_lock);
6549         for (i = 0; i < bp->cp_nr_rings; i++) {
6550                 struct bnxt_napi *bnapi = bp->bnapi[i];
6551                 struct bnxt_coal *hw_coal;
6552                 u16 ring_id;
6553
6554                 req = &req_rx;
6555                 if (!bnapi->rx_ring) {
6556                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6557                         req = &req_tx;
6558                 } else {
6559                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6560                 }
6561                 req->ring_id = cpu_to_le16(ring_id);
6562
6563                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6564                                         HWRM_CMD_TIMEOUT);
6565                 if (rc)
6566                         break;
6567
6568                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6569                         continue;
6570
6571                 if (bnapi->rx_ring && bnapi->tx_ring) {
6572                         req = &req_tx;
6573                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6574                         req->ring_id = cpu_to_le16(ring_id);
6575                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6576                                                 HWRM_CMD_TIMEOUT);
6577                         if (rc)
6578                                 break;
6579                 }
6580                 if (bnapi->rx_ring)
6581                         hw_coal = &bp->rx_coal;
6582                 else
6583                         hw_coal = &bp->tx_coal;
6584                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6585         }
6586         mutex_unlock(&bp->hwrm_cmd_lock);
6587         return rc;
6588 }
6589
6590 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6591 {
6592         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6593         struct hwrm_stat_ctx_free_input req = {0};
6594         int i;
6595
6596         if (!bp->bnapi)
6597                 return;
6598
6599         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6600                 return;
6601
6602         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6603         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6604
6605         mutex_lock(&bp->hwrm_cmd_lock);
6606         for (i = 0; i < bp->cp_nr_rings; i++) {
6607                 struct bnxt_napi *bnapi = bp->bnapi[i];
6608                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6609
6610                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6611                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6612                         if (BNXT_FW_MAJ(bp) <= 20) {
6613                                 req0.stat_ctx_id = req.stat_ctx_id;
6614                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6615                                                    HWRM_CMD_TIMEOUT);
6616                         }
6617                         _hwrm_send_message(bp, &req, sizeof(req),
6618                                            HWRM_CMD_TIMEOUT);
6619
6620                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6621                 }
6622         }
6623         mutex_unlock(&bp->hwrm_cmd_lock);
6624 }
6625
6626 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6627 {
6628         int rc = 0, i;
6629         struct hwrm_stat_ctx_alloc_input req = {0};
6630         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6631
6632         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6633                 return 0;
6634
6635         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6636
6637         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6638         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6639
6640         mutex_lock(&bp->hwrm_cmd_lock);
6641         for (i = 0; i < bp->cp_nr_rings; i++) {
6642                 struct bnxt_napi *bnapi = bp->bnapi[i];
6643                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6644
6645                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6646
6647                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6648                                         HWRM_CMD_TIMEOUT);
6649                 if (rc)
6650                         break;
6651
6652                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6653
6654                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6655         }
6656         mutex_unlock(&bp->hwrm_cmd_lock);
6657         return rc;
6658 }
6659
6660 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6661 {
6662         struct hwrm_func_qcfg_input req = {0};
6663         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6664         u32 min_db_offset = 0;
6665         u16 flags;
6666         int rc;
6667
6668         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6669         req.fid = cpu_to_le16(0xffff);
6670         mutex_lock(&bp->hwrm_cmd_lock);
6671         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6672         if (rc)
6673                 goto func_qcfg_exit;
6674
6675 #ifdef CONFIG_BNXT_SRIOV
6676         if (BNXT_VF(bp)) {
6677                 struct bnxt_vf_info *vf = &bp->vf;
6678
6679                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6680         } else {
6681                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6682         }
6683 #endif
6684         flags = le16_to_cpu(resp->flags);
6685         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6686                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6687                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6688                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6689                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6690         }
6691         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6692                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6693         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6694                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6695
6696         switch (resp->port_partition_type) {
6697         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6698         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6699         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6700                 bp->port_partition_type = resp->port_partition_type;
6701                 break;
6702         }
6703         if (bp->hwrm_spec_code < 0x10707 ||
6704             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6705                 bp->br_mode = BRIDGE_MODE_VEB;
6706         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6707                 bp->br_mode = BRIDGE_MODE_VEPA;
6708         else
6709                 bp->br_mode = BRIDGE_MODE_UNDEF;
6710
6711         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6712         if (!bp->max_mtu)
6713                 bp->max_mtu = BNXT_MAX_MTU;
6714
6715         if (bp->db_size)
6716                 goto func_qcfg_exit;
6717
6718         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6719                 if (BNXT_PF(bp))
6720                         min_db_offset = DB_PF_OFFSET_P5;
6721                 else
6722                         min_db_offset = DB_VF_OFFSET_P5;
6723         }
6724         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6725                                  1024);
6726         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6727             bp->db_size <= min_db_offset)
6728                 bp->db_size = pci_resource_len(bp->pdev, 2);
6729
6730 func_qcfg_exit:
6731         mutex_unlock(&bp->hwrm_cmd_lock);
6732         return rc;
6733 }
6734
6735 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6736 {
6737         struct hwrm_func_backing_store_qcaps_input req = {0};
6738         struct hwrm_func_backing_store_qcaps_output *resp =
6739                 bp->hwrm_cmd_resp_addr;
6740         int rc;
6741
6742         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6743                 return 0;
6744
6745         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6746         mutex_lock(&bp->hwrm_cmd_lock);
6747         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6748         if (!rc) {
6749                 struct bnxt_ctx_pg_info *ctx_pg;
6750                 struct bnxt_ctx_mem_info *ctx;
6751                 int i, tqm_rings;
6752
6753                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6754                 if (!ctx) {
6755                         rc = -ENOMEM;
6756                         goto ctx_err;
6757                 }
6758                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6759                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6760                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6761                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6762                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6763                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6764                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6765                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6766                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6767                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6768                 ctx->vnic_max_vnic_entries =
6769                         le16_to_cpu(resp->vnic_max_vnic_entries);
6770                 ctx->vnic_max_ring_table_entries =
6771                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6772                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6773                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6774                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6775                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6776                 ctx->tqm_min_entries_per_ring =
6777                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6778                 ctx->tqm_max_entries_per_ring =
6779                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6780                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6781                 if (!ctx->tqm_entries_multiple)
6782                         ctx->tqm_entries_multiple = 1;
6783                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6784                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6785                 ctx->mrav_num_entries_units =
6786                         le16_to_cpu(resp->mrav_num_entries_units);
6787                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6788                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6789                 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6790                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6791                 if (!ctx->tqm_fp_rings_count)
6792                         ctx->tqm_fp_rings_count = bp->max_q;
6793
6794                 tqm_rings = ctx->tqm_fp_rings_count + 1;
6795                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6796                 if (!ctx_pg) {
6797                         kfree(ctx);
6798                         rc = -ENOMEM;
6799                         goto ctx_err;
6800                 }
6801                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6802                         ctx->tqm_mem[i] = ctx_pg;
6803                 bp->ctx = ctx;
6804         } else {
6805                 rc = 0;
6806         }
6807 ctx_err:
6808         mutex_unlock(&bp->hwrm_cmd_lock);
6809         return rc;
6810 }
6811
6812 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6813                                   __le64 *pg_dir)
6814 {
6815         u8 pg_size = 0;
6816
6817         if (BNXT_PAGE_SHIFT == 13)
6818                 pg_size = 1 << 4;
6819         else if (BNXT_PAGE_SIZE == 16)
6820                 pg_size = 2 << 4;
6821
6822         *pg_attr = pg_size;
6823         if (rmem->depth >= 1) {
6824                 if (rmem->depth == 2)
6825                         *pg_attr |= 2;
6826                 else
6827                         *pg_attr |= 1;
6828                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6829         } else {
6830                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6831         }
6832 }
6833
6834 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6835         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6836          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6837          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6838          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6839          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6840
6841 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6842 {
6843         struct hwrm_func_backing_store_cfg_input req = {0};
6844         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6845         struct bnxt_ctx_pg_info *ctx_pg;
6846         __le32 *num_entries;
6847         __le64 *pg_dir;
6848         u32 flags = 0;
6849         u8 *pg_attr;
6850         u32 ena;
6851         int i;
6852
6853         if (!ctx)
6854                 return 0;
6855
6856         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6857         req.enables = cpu_to_le32(enables);
6858
6859         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6860                 ctx_pg = &ctx->qp_mem;
6861                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6862                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6863                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6864                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6865                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6866                                       &req.qpc_pg_size_qpc_lvl,
6867                                       &req.qpc_page_dir);
6868         }
6869         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6870                 ctx_pg = &ctx->srq_mem;
6871                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6872                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6873                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6874                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6875                                       &req.srq_pg_size_srq_lvl,
6876                                       &req.srq_page_dir);
6877         }
6878         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6879                 ctx_pg = &ctx->cq_mem;
6880                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6881                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6882                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6883                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6884                                       &req.cq_page_dir);
6885         }
6886         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6887                 ctx_pg = &ctx->vnic_mem;
6888                 req.vnic_num_vnic_entries =
6889                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6890                 req.vnic_num_ring_table_entries =
6891                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6892                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6893                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6894                                       &req.vnic_pg_size_vnic_lvl,
6895                                       &req.vnic_page_dir);
6896         }
6897         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6898                 ctx_pg = &ctx->stat_mem;
6899                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6900                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6901                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6902                                       &req.stat_pg_size_stat_lvl,
6903                                       &req.stat_page_dir);
6904         }
6905         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6906                 ctx_pg = &ctx->mrav_mem;
6907                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6908                 if (ctx->mrav_num_entries_units)
6909                         flags |=
6910                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6911                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6912                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6913                                       &req.mrav_pg_size_mrav_lvl,
6914                                       &req.mrav_page_dir);
6915         }
6916         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6917                 ctx_pg = &ctx->tim_mem;
6918                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6919                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6920                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6921                                       &req.tim_pg_size_tim_lvl,
6922                                       &req.tim_page_dir);
6923         }
6924         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6925              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6926              pg_dir = &req.tqm_sp_page_dir,
6927              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6928              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6929                 if (!(enables & ena))
6930                         continue;
6931
6932                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6933                 ctx_pg = ctx->tqm_mem[i];
6934                 *num_entries = cpu_to_le32(ctx_pg->entries);
6935                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6936         }
6937         req.flags = cpu_to_le32(flags);
6938         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6939 }
6940
6941 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6942                                   struct bnxt_ctx_pg_info *ctx_pg)
6943 {
6944         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6945
6946         rmem->page_size = BNXT_PAGE_SIZE;
6947         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6948         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6949         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6950         if (rmem->depth >= 1)
6951                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6952         return bnxt_alloc_ring(bp, rmem);
6953 }
6954
6955 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6956                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6957                                   u8 depth, bool use_init_val)
6958 {
6959         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6960         int rc;
6961
6962         if (!mem_size)
6963                 return -EINVAL;
6964
6965         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6966         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6967                 ctx_pg->nr_pages = 0;
6968                 return -EINVAL;
6969         }
6970         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6971                 int nr_tbls, i;
6972
6973                 rmem->depth = 2;
6974                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6975                                              GFP_KERNEL);
6976                 if (!ctx_pg->ctx_pg_tbl)
6977                         return -ENOMEM;
6978                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6979                 rmem->nr_pages = nr_tbls;
6980                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6981                 if (rc)
6982                         return rc;
6983                 for (i = 0; i < nr_tbls; i++) {
6984                         struct bnxt_ctx_pg_info *pg_tbl;
6985
6986                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6987                         if (!pg_tbl)
6988                                 return -ENOMEM;
6989                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6990                         rmem = &pg_tbl->ring_mem;
6991                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6992                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6993                         rmem->depth = 1;
6994                         rmem->nr_pages = MAX_CTX_PAGES;
6995                         if (use_init_val)
6996                                 rmem->init_val = bp->ctx->ctx_kind_initializer;
6997                         if (i == (nr_tbls - 1)) {
6998                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6999
7000                                 if (rem)
7001                                         rmem->nr_pages = rem;
7002                         }
7003                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7004                         if (rc)
7005                                 break;
7006                 }
7007         } else {
7008                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7009                 if (rmem->nr_pages > 1 || depth)
7010                         rmem->depth = 1;
7011                 if (use_init_val)
7012                         rmem->init_val = bp->ctx->ctx_kind_initializer;
7013                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7014         }
7015         return rc;
7016 }
7017
7018 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7019                                   struct bnxt_ctx_pg_info *ctx_pg)
7020 {
7021         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7022
7023         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7024             ctx_pg->ctx_pg_tbl) {
7025                 int i, nr_tbls = rmem->nr_pages;
7026
7027                 for (i = 0; i < nr_tbls; i++) {
7028                         struct bnxt_ctx_pg_info *pg_tbl;
7029                         struct bnxt_ring_mem_info *rmem2;
7030
7031                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7032                         if (!pg_tbl)
7033                                 continue;
7034                         rmem2 = &pg_tbl->ring_mem;
7035                         bnxt_free_ring(bp, rmem2);
7036                         ctx_pg->ctx_pg_arr[i] = NULL;
7037                         kfree(pg_tbl);
7038                         ctx_pg->ctx_pg_tbl[i] = NULL;
7039                 }
7040                 kfree(ctx_pg->ctx_pg_tbl);
7041                 ctx_pg->ctx_pg_tbl = NULL;
7042         }
7043         bnxt_free_ring(bp, rmem);
7044         ctx_pg->nr_pages = 0;
7045 }
7046
7047 static void bnxt_free_ctx_mem(struct bnxt *bp)
7048 {
7049         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7050         int i;
7051
7052         if (!ctx)
7053                 return;
7054
7055         if (ctx->tqm_mem[0]) {
7056                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7057                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7058                 kfree(ctx->tqm_mem[0]);
7059                 ctx->tqm_mem[0] = NULL;
7060         }
7061
7062         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7063         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7064         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7065         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7066         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7067         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7068         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7069         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7070 }
7071
7072 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7073 {
7074         struct bnxt_ctx_pg_info *ctx_pg;
7075         struct bnxt_ctx_mem_info *ctx;
7076         u32 mem_size, ena, entries;
7077         u32 entries_sp, min;
7078         u32 num_mr, num_ah;
7079         u32 extra_srqs = 0;
7080         u32 extra_qps = 0;
7081         u8 pg_lvl = 1;
7082         int i, rc;
7083
7084         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7085         if (rc) {
7086                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7087                            rc);
7088                 return rc;
7089         }
7090         ctx = bp->ctx;
7091         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7092                 return 0;
7093
7094         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7095                 pg_lvl = 2;
7096                 extra_qps = 65536;
7097                 extra_srqs = 8192;
7098         }
7099
7100         ctx_pg = &ctx->qp_mem;
7101         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7102                           extra_qps;
7103         mem_size = ctx->qp_entry_size * ctx_pg->entries;
7104         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7105         if (rc)
7106                 return rc;
7107
7108         ctx_pg = &ctx->srq_mem;
7109         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7110         mem_size = ctx->srq_entry_size * ctx_pg->entries;
7111         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7112         if (rc)
7113                 return rc;
7114
7115         ctx_pg = &ctx->cq_mem;
7116         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7117         mem_size = ctx->cq_entry_size * ctx_pg->entries;
7118         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7119         if (rc)
7120                 return rc;
7121
7122         ctx_pg = &ctx->vnic_mem;
7123         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7124                           ctx->vnic_max_ring_table_entries;
7125         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7126         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7127         if (rc)
7128                 return rc;
7129
7130         ctx_pg = &ctx->stat_mem;
7131         ctx_pg->entries = ctx->stat_max_entries;
7132         mem_size = ctx->stat_entry_size * ctx_pg->entries;
7133         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7134         if (rc)
7135                 return rc;
7136
7137         ena = 0;
7138         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7139                 goto skip_rdma;
7140
7141         ctx_pg = &ctx->mrav_mem;
7142         /* 128K extra is needed to accommodate static AH context
7143          * allocation by f/w.
7144          */
7145         num_mr = 1024 * 256;
7146         num_ah = 1024 * 128;
7147         ctx_pg->entries = num_mr + num_ah;
7148         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7149         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7150         if (rc)
7151                 return rc;
7152         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7153         if (ctx->mrav_num_entries_units)
7154                 ctx_pg->entries =
7155                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7156                          (num_ah / ctx->mrav_num_entries_units);
7157
7158         ctx_pg = &ctx->tim_mem;
7159         ctx_pg->entries = ctx->qp_mem.entries;
7160         mem_size = ctx->tim_entry_size * ctx_pg->entries;
7161         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7162         if (rc)
7163                 return rc;
7164         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7165
7166 skip_rdma:
7167         min = ctx->tqm_min_entries_per_ring;
7168         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7169                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7170         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7171         entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7172         entries = roundup(entries, ctx->tqm_entries_multiple);
7173         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7174         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7175                 ctx_pg = ctx->tqm_mem[i];
7176                 ctx_pg->entries = i ? entries : entries_sp;
7177                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7178                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7179                 if (rc)
7180                         return rc;
7181                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7182         }
7183         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7184         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7185         if (rc) {
7186                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7187                            rc);
7188                 return rc;
7189         }
7190         ctx->flags |= BNXT_CTX_FLAG_INITED;
7191         return 0;
7192 }
7193
7194 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7195 {
7196         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7197         struct hwrm_func_resource_qcaps_input req = {0};
7198         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7199         int rc;
7200
7201         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7202         req.fid = cpu_to_le16(0xffff);
7203
7204         mutex_lock(&bp->hwrm_cmd_lock);
7205         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7206                                        HWRM_CMD_TIMEOUT);
7207         if (rc)
7208                 goto hwrm_func_resc_qcaps_exit;
7209
7210         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7211         if (!all)
7212                 goto hwrm_func_resc_qcaps_exit;
7213
7214         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7215         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7216         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7217         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7218         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7219         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7220         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7221         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7222         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7223         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7224         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7225         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7226         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7227         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7228         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7229         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7230
7231         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7232                 u16 max_msix = le16_to_cpu(resp->max_msix);
7233
7234                 hw_resc->max_nqs = max_msix;
7235                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7236         }
7237
7238         if (BNXT_PF(bp)) {
7239                 struct bnxt_pf_info *pf = &bp->pf;
7240
7241                 pf->vf_resv_strategy =
7242                         le16_to_cpu(resp->vf_reservation_strategy);
7243                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7244                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7245         }
7246 hwrm_func_resc_qcaps_exit:
7247         mutex_unlock(&bp->hwrm_cmd_lock);
7248         return rc;
7249 }
7250
7251 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7252 {
7253         int rc = 0;
7254         struct hwrm_func_qcaps_input req = {0};
7255         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7256         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7257         u32 flags, flags_ext;
7258
7259         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7260         req.fid = cpu_to_le16(0xffff);
7261
7262         mutex_lock(&bp->hwrm_cmd_lock);
7263         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7264         if (rc)
7265                 goto hwrm_func_qcaps_exit;
7266
7267         flags = le32_to_cpu(resp->flags);
7268         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7269                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7270         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7271                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7272         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7273                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7274         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7275                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7276         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7277                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7278         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7279                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7280         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7281                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7282         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7283                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7284
7285         flags_ext = le32_to_cpu(resp->flags_ext);
7286         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7287                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7288
7289         bp->tx_push_thresh = 0;
7290         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7291             BNXT_FW_MAJ(bp) > 217)
7292                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7293
7294         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7295         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7296         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7297         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7298         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7299         if (!hw_resc->max_hw_ring_grps)
7300                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7301         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7302         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7303         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7304
7305         if (BNXT_PF(bp)) {
7306                 struct bnxt_pf_info *pf = &bp->pf;
7307
7308                 pf->fw_fid = le16_to_cpu(resp->fid);
7309                 pf->port_id = le16_to_cpu(resp->port_id);
7310                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7311                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7312                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7313                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7314                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7315                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7316                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7317                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7318                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7319                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7320                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7321                         bp->flags |= BNXT_FLAG_WOL_CAP;
7322         } else {
7323 #ifdef CONFIG_BNXT_SRIOV
7324                 struct bnxt_vf_info *vf = &bp->vf;
7325
7326                 vf->fw_fid = le16_to_cpu(resp->fid);
7327                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7328 #endif
7329         }
7330
7331 hwrm_func_qcaps_exit:
7332         mutex_unlock(&bp->hwrm_cmd_lock);
7333         return rc;
7334 }
7335
7336 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7337
7338 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7339 {
7340         int rc;
7341
7342         rc = __bnxt_hwrm_func_qcaps(bp);
7343         if (rc)
7344                 return rc;
7345         rc = bnxt_hwrm_queue_qportcfg(bp);
7346         if (rc) {
7347                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7348                 return rc;
7349         }
7350         if (bp->hwrm_spec_code >= 0x10803) {
7351                 rc = bnxt_alloc_ctx_mem(bp);
7352                 if (rc)
7353                         return rc;
7354                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7355                 if (!rc)
7356                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7357         }
7358         return 0;
7359 }
7360
7361 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7362 {
7363         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7364         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7365         int rc = 0;
7366         u32 flags;
7367
7368         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7369                 return 0;
7370
7371         resp = bp->hwrm_cmd_resp_addr;
7372         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7373
7374         mutex_lock(&bp->hwrm_cmd_lock);
7375         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7376         if (rc)
7377                 goto hwrm_cfa_adv_qcaps_exit;
7378
7379         flags = le32_to_cpu(resp->flags);
7380         if (flags &
7381             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7382                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7383
7384 hwrm_cfa_adv_qcaps_exit:
7385         mutex_unlock(&bp->hwrm_cmd_lock);
7386         return rc;
7387 }
7388
7389 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7390 {
7391         if (bp->fw_health)
7392                 return 0;
7393
7394         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7395         if (!bp->fw_health)
7396                 return -ENOMEM;
7397
7398         return 0;
7399 }
7400
7401 static int bnxt_alloc_fw_health(struct bnxt *bp)
7402 {
7403         int rc;
7404
7405         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7406             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7407                 return 0;
7408
7409         rc = __bnxt_alloc_fw_health(bp);
7410         if (rc) {
7411                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7412                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7413                 return rc;
7414         }
7415
7416         return 0;
7417 }
7418
7419 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7420 {
7421         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7422                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7423                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7424 }
7425
7426 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7427 {
7428         void __iomem *hs;
7429         u32 status_loc;
7430         u32 reg_type;
7431         u32 sig;
7432
7433         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7434         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7435
7436         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7437         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7438                 if (bp->fw_health)
7439                         bp->fw_health->status_reliable = false;
7440                 return;
7441         }
7442
7443         if (__bnxt_alloc_fw_health(bp)) {
7444                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7445                 return;
7446         }
7447
7448         status_loc = readl(hs + offsetof(struct hcomm_status, fw_status_loc));
7449         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7450         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7451         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7452                 __bnxt_map_fw_health_reg(bp, status_loc);
7453                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7454                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7455         }
7456
7457         bp->fw_health->status_reliable = true;
7458 }
7459
7460 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7461 {
7462         struct bnxt_fw_health *fw_health = bp->fw_health;
7463         u32 reg_base = 0xffffffff;
7464         int i;
7465
7466         /* Only pre-map the monitoring GRC registers using window 3 */
7467         for (i = 0; i < 4; i++) {
7468                 u32 reg = fw_health->regs[i];
7469
7470                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7471                         continue;
7472                 if (reg_base == 0xffffffff)
7473                         reg_base = reg & BNXT_GRC_BASE_MASK;
7474                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7475                         return -ERANGE;
7476                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7477         }
7478         if (reg_base == 0xffffffff)
7479                 return 0;
7480
7481         __bnxt_map_fw_health_reg(bp, reg_base);
7482         return 0;
7483 }
7484
7485 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7486 {
7487         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7488         struct bnxt_fw_health *fw_health = bp->fw_health;
7489         struct hwrm_error_recovery_qcfg_input req = {0};
7490         int rc, i;
7491
7492         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7493                 return 0;
7494
7495         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7496         mutex_lock(&bp->hwrm_cmd_lock);
7497         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7498         if (rc)
7499                 goto err_recovery_out;
7500         fw_health->flags = le32_to_cpu(resp->flags);
7501         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7502             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7503                 rc = -EINVAL;
7504                 goto err_recovery_out;
7505         }
7506         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7507         fw_health->master_func_wait_dsecs =
7508                 le32_to_cpu(resp->master_func_wait_period);
7509         fw_health->normal_func_wait_dsecs =
7510                 le32_to_cpu(resp->normal_func_wait_period);
7511         fw_health->post_reset_wait_dsecs =
7512                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7513         fw_health->post_reset_max_wait_dsecs =
7514                 le32_to_cpu(resp->max_bailout_time_after_reset);
7515         fw_health->regs[BNXT_FW_HEALTH_REG] =
7516                 le32_to_cpu(resp->fw_health_status_reg);
7517         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7518                 le32_to_cpu(resp->fw_heartbeat_reg);
7519         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7520                 le32_to_cpu(resp->fw_reset_cnt_reg);
7521         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7522                 le32_to_cpu(resp->reset_inprogress_reg);
7523         fw_health->fw_reset_inprog_reg_mask =
7524                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7525         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7526         if (fw_health->fw_reset_seq_cnt >= 16) {
7527                 rc = -EINVAL;
7528                 goto err_recovery_out;
7529         }
7530         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7531                 fw_health->fw_reset_seq_regs[i] =
7532                         le32_to_cpu(resp->reset_reg[i]);
7533                 fw_health->fw_reset_seq_vals[i] =
7534                         le32_to_cpu(resp->reset_reg_val[i]);
7535                 fw_health->fw_reset_seq_delay_msec[i] =
7536                         resp->delay_after_reset[i];
7537         }
7538 err_recovery_out:
7539         mutex_unlock(&bp->hwrm_cmd_lock);
7540         if (!rc)
7541                 rc = bnxt_map_fw_health_regs(bp);
7542         if (rc)
7543                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7544         return rc;
7545 }
7546
7547 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7548 {
7549         struct hwrm_func_reset_input req = {0};
7550
7551         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7552         req.enables = 0;
7553
7554         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7555 }
7556
7557 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7558 {
7559         struct hwrm_nvm_get_dev_info_output nvm_info;
7560
7561         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7562                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7563                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7564                          nvm_info.nvm_cfg_ver_upd);
7565 }
7566
7567 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7568 {
7569         int rc = 0;
7570         struct hwrm_queue_qportcfg_input req = {0};
7571         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7572         u8 i, j, *qptr;
7573         bool no_rdma;
7574
7575         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7576
7577         mutex_lock(&bp->hwrm_cmd_lock);
7578         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7579         if (rc)
7580                 goto qportcfg_exit;
7581
7582         if (!resp->max_configurable_queues) {
7583                 rc = -EINVAL;
7584                 goto qportcfg_exit;
7585         }
7586         bp->max_tc = resp->max_configurable_queues;
7587         bp->max_lltc = resp->max_configurable_lossless_queues;
7588         if (bp->max_tc > BNXT_MAX_QUEUE)
7589                 bp->max_tc = BNXT_MAX_QUEUE;
7590
7591         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7592         qptr = &resp->queue_id0;
7593         for (i = 0, j = 0; i < bp->max_tc; i++) {
7594                 bp->q_info[j].queue_id = *qptr;
7595                 bp->q_ids[i] = *qptr++;
7596                 bp->q_info[j].queue_profile = *qptr++;
7597                 bp->tc_to_qidx[j] = j;
7598                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7599                     (no_rdma && BNXT_PF(bp)))
7600                         j++;
7601         }
7602         bp->max_q = bp->max_tc;
7603         bp->max_tc = max_t(u8, j, 1);
7604
7605         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7606                 bp->max_tc = 1;
7607
7608         if (bp->max_lltc > bp->max_tc)
7609                 bp->max_lltc = bp->max_tc;
7610
7611 qportcfg_exit:
7612         mutex_unlock(&bp->hwrm_cmd_lock);
7613         return rc;
7614 }
7615
7616 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7617 {
7618         struct hwrm_ver_get_input req = {0};
7619         int rc;
7620
7621         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7622         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7623         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7624         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7625
7626         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7627                                    silent);
7628         return rc;
7629 }
7630
7631 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7632 {
7633         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7634         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7635         u32 dev_caps_cfg, hwrm_ver;
7636         int rc, len;
7637
7638         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7639         mutex_lock(&bp->hwrm_cmd_lock);
7640         rc = __bnxt_hwrm_ver_get(bp, false);
7641         if (rc)
7642                 goto hwrm_ver_get_exit;
7643
7644         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7645
7646         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7647                              resp->hwrm_intf_min_8b << 8 |
7648                              resp->hwrm_intf_upd_8b;
7649         if (resp->hwrm_intf_maj_8b < 1) {
7650                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7651                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7652                             resp->hwrm_intf_upd_8b);
7653                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7654         }
7655
7656         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7657                         HWRM_VERSION_UPDATE;
7658
7659         if (bp->hwrm_spec_code > hwrm_ver)
7660                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7661                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7662                          HWRM_VERSION_UPDATE);
7663         else
7664                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7665                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7666                          resp->hwrm_intf_upd_8b);
7667
7668         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7669         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7670                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7671                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7672                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7673                 len = FW_VER_STR_LEN;
7674         } else {
7675                 fw_maj = resp->hwrm_fw_maj_8b;
7676                 fw_min = resp->hwrm_fw_min_8b;
7677                 fw_bld = resp->hwrm_fw_bld_8b;
7678                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7679                 len = BC_HWRM_STR_LEN;
7680         }
7681         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7682         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7683                  fw_rsv);
7684
7685         if (strlen(resp->active_pkg_name)) {
7686                 int fw_ver_len = strlen(bp->fw_ver_str);
7687
7688                 snprintf(bp->fw_ver_str + fw_ver_len,
7689                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7690                          resp->active_pkg_name);
7691                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7692         }
7693
7694         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7695         if (!bp->hwrm_cmd_timeout)
7696                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7697
7698         if (resp->hwrm_intf_maj_8b >= 1) {
7699                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7700                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7701         }
7702         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7703                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7704
7705         bp->chip_num = le16_to_cpu(resp->chip_num);
7706         bp->chip_rev = resp->chip_rev;
7707         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7708             !resp->chip_metal)
7709                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7710
7711         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7712         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7713             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7714                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7715
7716         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7717                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7718
7719         if (dev_caps_cfg &
7720             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7721                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7722
7723         if (dev_caps_cfg &
7724             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7725                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7726
7727         if (dev_caps_cfg &
7728             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7729                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7730
7731 hwrm_ver_get_exit:
7732         mutex_unlock(&bp->hwrm_cmd_lock);
7733         return rc;
7734 }
7735
7736 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7737 {
7738         struct hwrm_fw_set_time_input req = {0};
7739         struct tm tm;
7740         time64_t now = ktime_get_real_seconds();
7741
7742         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7743             bp->hwrm_spec_code < 0x10400)
7744                 return -EOPNOTSUPP;
7745
7746         time64_to_tm(now, 0, &tm);
7747         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7748         req.year = cpu_to_le16(1900 + tm.tm_year);
7749         req.month = 1 + tm.tm_mon;
7750         req.day = tm.tm_mday;
7751         req.hour = tm.tm_hour;
7752         req.minute = tm.tm_min;
7753         req.second = tm.tm_sec;
7754         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7755 }
7756
7757 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7758 {
7759         u64 sw_tmp;
7760
7761         sw_tmp = (*sw & ~mask) | hw;
7762         if (hw < (*sw & mask))
7763                 sw_tmp += mask + 1;
7764         WRITE_ONCE(*sw, sw_tmp);
7765 }
7766
7767 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7768                                     int count, bool ignore_zero)
7769 {
7770         int i;
7771
7772         for (i = 0; i < count; i++) {
7773                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7774
7775                 if (ignore_zero && !hw)
7776                         continue;
7777
7778                 if (masks[i] == -1ULL)
7779                         sw_stats[i] = hw;
7780                 else
7781                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7782         }
7783 }
7784
7785 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7786 {
7787         if (!stats->hw_stats)
7788                 return;
7789
7790         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7791                                 stats->hw_masks, stats->len / 8, false);
7792 }
7793
7794 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7795 {
7796         struct bnxt_stats_mem *ring0_stats;
7797         bool ignore_zero = false;
7798         int i;
7799
7800         /* Chip bug.  Counter intermittently becomes 0. */
7801         if (bp->flags & BNXT_FLAG_CHIP_P5)
7802                 ignore_zero = true;
7803
7804         for (i = 0; i < bp->cp_nr_rings; i++) {
7805                 struct bnxt_napi *bnapi = bp->bnapi[i];
7806                 struct bnxt_cp_ring_info *cpr;
7807                 struct bnxt_stats_mem *stats;
7808
7809                 cpr = &bnapi->cp_ring;
7810                 stats = &cpr->stats;
7811                 if (!i)
7812                         ring0_stats = stats;
7813                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7814                                         ring0_stats->hw_masks,
7815                                         ring0_stats->len / 8, ignore_zero);
7816         }
7817         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7818                 struct bnxt_stats_mem *stats = &bp->port_stats;
7819                 __le64 *hw_stats = stats->hw_stats;
7820                 u64 *sw_stats = stats->sw_stats;
7821                 u64 *masks = stats->hw_masks;
7822                 int cnt;
7823
7824                 cnt = sizeof(struct rx_port_stats) / 8;
7825                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7826
7827                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7828                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7829                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7830                 cnt = sizeof(struct tx_port_stats) / 8;
7831                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7832         }
7833         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7834                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7835                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7836         }
7837 }
7838
7839 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7840 {
7841         struct bnxt_pf_info *pf = &bp->pf;
7842         struct hwrm_port_qstats_input req = {0};
7843
7844         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7845                 return 0;
7846
7847         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7848                 return -EOPNOTSUPP;
7849
7850         req.flags = flags;
7851         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7852         req.port_id = cpu_to_le16(pf->port_id);
7853         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7854                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
7855         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7856         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7857 }
7858
7859 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7860 {
7861         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7862         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7863         struct hwrm_port_qstats_ext_input req = {0};
7864         struct bnxt_pf_info *pf = &bp->pf;
7865         u32 tx_stat_size;
7866         int rc;
7867
7868         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7869                 return 0;
7870
7871         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7872                 return -EOPNOTSUPP;
7873
7874         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7875         req.flags = flags;
7876         req.port_id = cpu_to_le16(pf->port_id);
7877         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7878         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7879         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7880                        sizeof(struct tx_port_stats_ext) : 0;
7881         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7882         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7883         mutex_lock(&bp->hwrm_cmd_lock);
7884         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7885         if (!rc) {
7886                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7887                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7888                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7889         } else {
7890                 bp->fw_rx_stats_ext_size = 0;
7891                 bp->fw_tx_stats_ext_size = 0;
7892         }
7893         if (flags)
7894                 goto qstats_done;
7895
7896         if (bp->fw_tx_stats_ext_size <=
7897             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7898                 mutex_unlock(&bp->hwrm_cmd_lock);
7899                 bp->pri2cos_valid = 0;
7900                 return rc;
7901         }
7902
7903         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7904         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7905
7906         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7907         if (!rc) {
7908                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7909                 u8 *pri2cos;
7910                 int i, j;
7911
7912                 resp2 = bp->hwrm_cmd_resp_addr;
7913                 pri2cos = &resp2->pri0_cos_queue_id;
7914                 for (i = 0; i < 8; i++) {
7915                         u8 queue_id = pri2cos[i];
7916                         u8 queue_idx;
7917
7918                         /* Per port queue IDs start from 0, 10, 20, etc */
7919                         queue_idx = queue_id % 10;
7920                         if (queue_idx > BNXT_MAX_QUEUE) {
7921                                 bp->pri2cos_valid = false;
7922                                 goto qstats_done;
7923                         }
7924                         for (j = 0; j < bp->max_q; j++) {
7925                                 if (bp->q_ids[j] == queue_id)
7926                                         bp->pri2cos_idx[i] = queue_idx;
7927                         }
7928                 }
7929                 bp->pri2cos_valid = 1;
7930         }
7931 qstats_done:
7932         mutex_unlock(&bp->hwrm_cmd_lock);
7933         return rc;
7934 }
7935
7936 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7937 {
7938         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7939                 bnxt_hwrm_tunnel_dst_port_free(
7940                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7941         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7942                 bnxt_hwrm_tunnel_dst_port_free(
7943                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7944 }
7945
7946 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7947 {
7948         int rc, i;
7949         u32 tpa_flags = 0;
7950
7951         if (set_tpa)
7952                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7953         else if (BNXT_NO_FW_ACCESS(bp))
7954                 return 0;
7955         for (i = 0; i < bp->nr_vnics; i++) {
7956                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7957                 if (rc) {
7958                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7959                                    i, rc);
7960                         return rc;
7961                 }
7962         }
7963         return 0;
7964 }
7965
7966 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7967 {
7968         int i;
7969
7970         for (i = 0; i < bp->nr_vnics; i++)
7971                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7972 }
7973
7974 static void bnxt_clear_vnic(struct bnxt *bp)
7975 {
7976         if (!bp->vnic_info)
7977                 return;
7978
7979         bnxt_hwrm_clear_vnic_filter(bp);
7980         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7981                 /* clear all RSS setting before free vnic ctx */
7982                 bnxt_hwrm_clear_vnic_rss(bp);
7983                 bnxt_hwrm_vnic_ctx_free(bp);
7984         }
7985         /* before free the vnic, undo the vnic tpa settings */
7986         if (bp->flags & BNXT_FLAG_TPA)
7987                 bnxt_set_tpa(bp, false);
7988         bnxt_hwrm_vnic_free(bp);
7989         if (bp->flags & BNXT_FLAG_CHIP_P5)
7990                 bnxt_hwrm_vnic_ctx_free(bp);
7991 }
7992
7993 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7994                                     bool irq_re_init)
7995 {
7996         bnxt_clear_vnic(bp);
7997         bnxt_hwrm_ring_free(bp, close_path);
7998         bnxt_hwrm_ring_grp_free(bp);
7999         if (irq_re_init) {
8000                 bnxt_hwrm_stat_ctx_free(bp);
8001                 bnxt_hwrm_free_tunnel_ports(bp);
8002         }
8003 }
8004
8005 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8006 {
8007         struct hwrm_func_cfg_input req = {0};
8008
8009         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8010         req.fid = cpu_to_le16(0xffff);
8011         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8012         if (br_mode == BRIDGE_MODE_VEB)
8013                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8014         else if (br_mode == BRIDGE_MODE_VEPA)
8015                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8016         else
8017                 return -EINVAL;
8018         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8019 }
8020
8021 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8022 {
8023         struct hwrm_func_cfg_input req = {0};
8024
8025         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8026                 return 0;
8027
8028         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8029         req.fid = cpu_to_le16(0xffff);
8030         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8031         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8032         if (size == 128)
8033                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8034
8035         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8036 }
8037
8038 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8039 {
8040         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8041         int rc;
8042
8043         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8044                 goto skip_rss_ctx;
8045
8046         /* allocate context for vnic */
8047         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8048         if (rc) {
8049                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8050                            vnic_id, rc);
8051                 goto vnic_setup_err;
8052         }
8053         bp->rsscos_nr_ctxs++;
8054
8055         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8056                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8057                 if (rc) {
8058                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8059                                    vnic_id, rc);
8060                         goto vnic_setup_err;
8061                 }
8062                 bp->rsscos_nr_ctxs++;
8063         }
8064
8065 skip_rss_ctx:
8066         /* configure default vnic, ring grp */
8067         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8068         if (rc) {
8069                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8070                            vnic_id, rc);
8071                 goto vnic_setup_err;
8072         }
8073
8074         /* Enable RSS hashing on vnic */
8075         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8076         if (rc) {
8077                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8078                            vnic_id, rc);
8079                 goto vnic_setup_err;
8080         }
8081
8082         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8083                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8084                 if (rc) {
8085                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8086                                    vnic_id, rc);
8087                 }
8088         }
8089
8090 vnic_setup_err:
8091         return rc;
8092 }
8093
8094 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8095 {
8096         int rc, i, nr_ctxs;
8097
8098         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8099         for (i = 0; i < nr_ctxs; i++) {
8100                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8101                 if (rc) {
8102                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8103                                    vnic_id, i, rc);
8104                         break;
8105                 }
8106                 bp->rsscos_nr_ctxs++;
8107         }
8108         if (i < nr_ctxs)
8109                 return -ENOMEM;
8110
8111         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8112         if (rc) {
8113                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8114                            vnic_id, rc);
8115                 return rc;
8116         }
8117         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8118         if (rc) {
8119                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8120                            vnic_id, rc);
8121                 return rc;
8122         }
8123         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8124                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8125                 if (rc) {
8126                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8127                                    vnic_id, rc);
8128                 }
8129         }
8130         return rc;
8131 }
8132
8133 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8134 {
8135         if (bp->flags & BNXT_FLAG_CHIP_P5)
8136                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8137         else
8138                 return __bnxt_setup_vnic(bp, vnic_id);
8139 }
8140
8141 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8142 {
8143 #ifdef CONFIG_RFS_ACCEL
8144         int i, rc = 0;
8145
8146         if (bp->flags & BNXT_FLAG_CHIP_P5)
8147                 return 0;
8148
8149         for (i = 0; i < bp->rx_nr_rings; i++) {
8150                 struct bnxt_vnic_info *vnic;
8151                 u16 vnic_id = i + 1;
8152                 u16 ring_id = i;
8153
8154                 if (vnic_id >= bp->nr_vnics)
8155                         break;
8156
8157                 vnic = &bp->vnic_info[vnic_id];
8158                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8159                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8160                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8161                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8162                 if (rc) {
8163                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8164                                    vnic_id, rc);
8165                         break;
8166                 }
8167                 rc = bnxt_setup_vnic(bp, vnic_id);
8168                 if (rc)
8169                         break;
8170         }
8171         return rc;
8172 #else
8173         return 0;
8174 #endif
8175 }
8176
8177 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8178 static bool bnxt_promisc_ok(struct bnxt *bp)
8179 {
8180 #ifdef CONFIG_BNXT_SRIOV
8181         if (BNXT_VF(bp) && !bp->vf.vlan)
8182                 return false;
8183 #endif
8184         return true;
8185 }
8186
8187 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8188 {
8189         unsigned int rc = 0;
8190
8191         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8192         if (rc) {
8193                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8194                            rc);
8195                 return rc;
8196         }
8197
8198         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8199         if (rc) {
8200                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8201                            rc);
8202                 return rc;
8203         }
8204         return rc;
8205 }
8206
8207 static int bnxt_cfg_rx_mode(struct bnxt *);
8208 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8209
8210 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8211 {
8212         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8213         int rc = 0;
8214         unsigned int rx_nr_rings = bp->rx_nr_rings;
8215
8216         if (irq_re_init) {
8217                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8218                 if (rc) {
8219                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8220                                    rc);
8221                         goto err_out;
8222                 }
8223         }
8224
8225         rc = bnxt_hwrm_ring_alloc(bp);
8226         if (rc) {
8227                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8228                 goto err_out;
8229         }
8230
8231         rc = bnxt_hwrm_ring_grp_alloc(bp);
8232         if (rc) {
8233                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8234                 goto err_out;
8235         }
8236
8237         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8238                 rx_nr_rings--;
8239
8240         /* default vnic 0 */
8241         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8242         if (rc) {
8243                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8244                 goto err_out;
8245         }
8246
8247         rc = bnxt_setup_vnic(bp, 0);
8248         if (rc)
8249                 goto err_out;
8250
8251         if (bp->flags & BNXT_FLAG_RFS) {
8252                 rc = bnxt_alloc_rfs_vnics(bp);
8253                 if (rc)
8254                         goto err_out;
8255         }
8256
8257         if (bp->flags & BNXT_FLAG_TPA) {
8258                 rc = bnxt_set_tpa(bp, true);
8259                 if (rc)
8260                         goto err_out;
8261         }
8262
8263         if (BNXT_VF(bp))
8264                 bnxt_update_vf_mac(bp);
8265
8266         /* Filter for default vnic 0 */
8267         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8268         if (rc) {
8269                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8270                 goto err_out;
8271         }
8272         vnic->uc_filter_count = 1;
8273
8274         vnic->rx_mask = 0;
8275         if (bp->dev->flags & IFF_BROADCAST)
8276                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8277
8278         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8279                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8280
8281         if (bp->dev->flags & IFF_ALLMULTI) {
8282                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8283                 vnic->mc_list_count = 0;
8284         } else {
8285                 u32 mask = 0;
8286
8287                 bnxt_mc_list_updated(bp, &mask);
8288                 vnic->rx_mask |= mask;
8289         }
8290
8291         rc = bnxt_cfg_rx_mode(bp);
8292         if (rc)
8293                 goto err_out;
8294
8295         rc = bnxt_hwrm_set_coal(bp);
8296         if (rc)
8297                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8298                                 rc);
8299
8300         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8301                 rc = bnxt_setup_nitroa0_vnic(bp);
8302                 if (rc)
8303                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8304                                    rc);
8305         }
8306
8307         if (BNXT_VF(bp)) {
8308                 bnxt_hwrm_func_qcfg(bp);
8309                 netdev_update_features(bp->dev);
8310         }
8311
8312         return 0;
8313
8314 err_out:
8315         bnxt_hwrm_resource_free(bp, 0, true);
8316
8317         return rc;
8318 }
8319
8320 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8321 {
8322         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8323         return 0;
8324 }
8325
8326 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8327 {
8328         bnxt_init_cp_rings(bp);
8329         bnxt_init_rx_rings(bp);
8330         bnxt_init_tx_rings(bp);
8331         bnxt_init_ring_grps(bp, irq_re_init);
8332         bnxt_init_vnics(bp);
8333
8334         return bnxt_init_chip(bp, irq_re_init);
8335 }
8336
8337 static int bnxt_set_real_num_queues(struct bnxt *bp)
8338 {
8339         int rc;
8340         struct net_device *dev = bp->dev;
8341
8342         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8343                                           bp->tx_nr_rings_xdp);
8344         if (rc)
8345                 return rc;
8346
8347         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8348         if (rc)
8349                 return rc;
8350
8351 #ifdef CONFIG_RFS_ACCEL
8352         if (bp->flags & BNXT_FLAG_RFS)
8353                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8354 #endif
8355
8356         return rc;
8357 }
8358
8359 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8360                            bool shared)
8361 {
8362         int _rx = *rx, _tx = *tx;
8363
8364         if (shared) {
8365                 *rx = min_t(int, _rx, max);
8366                 *tx = min_t(int, _tx, max);
8367         } else {
8368                 if (max < 2)
8369                         return -ENOMEM;
8370
8371                 while (_rx + _tx > max) {
8372                         if (_rx > _tx && _rx > 1)
8373                                 _rx--;
8374                         else if (_tx > 1)
8375                                 _tx--;
8376                 }
8377                 *rx = _rx;
8378                 *tx = _tx;
8379         }
8380         return 0;
8381 }
8382
8383 static void bnxt_setup_msix(struct bnxt *bp)
8384 {
8385         const int len = sizeof(bp->irq_tbl[0].name);
8386         struct net_device *dev = bp->dev;
8387         int tcs, i;
8388
8389         tcs = netdev_get_num_tc(dev);
8390         if (tcs) {
8391                 int i, off, count;
8392
8393                 for (i = 0; i < tcs; i++) {
8394                         count = bp->tx_nr_rings_per_tc;
8395                         off = i * count;
8396                         netdev_set_tc_queue(dev, i, count, off);
8397                 }
8398         }
8399
8400         for (i = 0; i < bp->cp_nr_rings; i++) {
8401                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8402                 char *attr;
8403
8404                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8405                         attr = "TxRx";
8406                 else if (i < bp->rx_nr_rings)
8407                         attr = "rx";
8408                 else
8409                         attr = "tx";
8410
8411                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8412                          attr, i);
8413                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8414         }
8415 }
8416
8417 static void bnxt_setup_inta(struct bnxt *bp)
8418 {
8419         const int len = sizeof(bp->irq_tbl[0].name);
8420
8421         if (netdev_get_num_tc(bp->dev))
8422                 netdev_reset_tc(bp->dev);
8423
8424         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8425                  0);
8426         bp->irq_tbl[0].handler = bnxt_inta;
8427 }
8428
8429 static int bnxt_setup_int_mode(struct bnxt *bp)
8430 {
8431         int rc;
8432
8433         if (bp->flags & BNXT_FLAG_USING_MSIX)
8434                 bnxt_setup_msix(bp);
8435         else
8436                 bnxt_setup_inta(bp);
8437
8438         rc = bnxt_set_real_num_queues(bp);
8439         return rc;
8440 }
8441
8442 #ifdef CONFIG_RFS_ACCEL
8443 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8444 {
8445         return bp->hw_resc.max_rsscos_ctxs;
8446 }
8447
8448 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8449 {
8450         return bp->hw_resc.max_vnics;
8451 }
8452 #endif
8453
8454 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8455 {
8456         return bp->hw_resc.max_stat_ctxs;
8457 }
8458
8459 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8460 {
8461         return bp->hw_resc.max_cp_rings;
8462 }
8463
8464 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8465 {
8466         unsigned int cp = bp->hw_resc.max_cp_rings;
8467
8468         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8469                 cp -= bnxt_get_ulp_msix_num(bp);
8470
8471         return cp;
8472 }
8473
8474 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8475 {
8476         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8477
8478         if (bp->flags & BNXT_FLAG_CHIP_P5)
8479                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8480
8481         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8482 }
8483
8484 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8485 {
8486         bp->hw_resc.max_irqs = max_irqs;
8487 }
8488
8489 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8490 {
8491         unsigned int cp;
8492
8493         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8494         if (bp->flags & BNXT_FLAG_CHIP_P5)
8495                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8496         else
8497                 return cp - bp->cp_nr_rings;
8498 }
8499
8500 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8501 {
8502         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8503 }
8504
8505 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8506 {
8507         int max_cp = bnxt_get_max_func_cp_rings(bp);
8508         int max_irq = bnxt_get_max_func_irqs(bp);
8509         int total_req = bp->cp_nr_rings + num;
8510         int max_idx, avail_msix;
8511
8512         max_idx = bp->total_irqs;
8513         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8514                 max_idx = min_t(int, bp->total_irqs, max_cp);
8515         avail_msix = max_idx - bp->cp_nr_rings;
8516         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8517                 return avail_msix;
8518
8519         if (max_irq < total_req) {
8520                 num = max_irq - bp->cp_nr_rings;
8521                 if (num <= 0)
8522                         return 0;
8523         }
8524         return num;
8525 }
8526
8527 static int bnxt_get_num_msix(struct bnxt *bp)
8528 {
8529         if (!BNXT_NEW_RM(bp))
8530                 return bnxt_get_max_func_irqs(bp);
8531
8532         return bnxt_nq_rings_in_use(bp);
8533 }
8534
8535 static int bnxt_init_msix(struct bnxt *bp)
8536 {
8537         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8538         struct msix_entry *msix_ent;
8539
8540         total_vecs = bnxt_get_num_msix(bp);
8541         max = bnxt_get_max_func_irqs(bp);
8542         if (total_vecs > max)
8543                 total_vecs = max;
8544
8545         if (!total_vecs)
8546                 return 0;
8547
8548         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8549         if (!msix_ent)
8550                 return -ENOMEM;
8551
8552         for (i = 0; i < total_vecs; i++) {
8553                 msix_ent[i].entry = i;
8554                 msix_ent[i].vector = 0;
8555         }
8556
8557         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8558                 min = 2;
8559
8560         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8561         ulp_msix = bnxt_get_ulp_msix_num(bp);
8562         if (total_vecs < 0 || total_vecs < ulp_msix) {
8563                 rc = -ENODEV;
8564                 goto msix_setup_exit;
8565         }
8566
8567         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8568         if (bp->irq_tbl) {
8569                 for (i = 0; i < total_vecs; i++)
8570                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8571
8572                 bp->total_irqs = total_vecs;
8573                 /* Trim rings based upon num of vectors allocated */
8574                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8575                                      total_vecs - ulp_msix, min == 1);
8576                 if (rc)
8577                         goto msix_setup_exit;
8578
8579                 bp->cp_nr_rings = (min == 1) ?
8580                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8581                                   bp->tx_nr_rings + bp->rx_nr_rings;
8582
8583         } else {
8584                 rc = -ENOMEM;
8585                 goto msix_setup_exit;
8586         }
8587         bp->flags |= BNXT_FLAG_USING_MSIX;
8588         kfree(msix_ent);
8589         return 0;
8590
8591 msix_setup_exit:
8592         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8593         kfree(bp->irq_tbl);
8594         bp->irq_tbl = NULL;
8595         pci_disable_msix(bp->pdev);
8596         kfree(msix_ent);
8597         return rc;
8598 }
8599
8600 static int bnxt_init_inta(struct bnxt *bp)
8601 {
8602         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8603         if (!bp->irq_tbl)
8604                 return -ENOMEM;
8605
8606         bp->total_irqs = 1;
8607         bp->rx_nr_rings = 1;
8608         bp->tx_nr_rings = 1;
8609         bp->cp_nr_rings = 1;
8610         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8611         bp->irq_tbl[0].vector = bp->pdev->irq;
8612         return 0;
8613 }
8614
8615 static int bnxt_init_int_mode(struct bnxt *bp)
8616 {
8617         int rc = 0;
8618
8619         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8620                 rc = bnxt_init_msix(bp);
8621
8622         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8623                 /* fallback to INTA */
8624                 rc = bnxt_init_inta(bp);
8625         }
8626         return rc;
8627 }
8628
8629 static void bnxt_clear_int_mode(struct bnxt *bp)
8630 {
8631         if (bp->flags & BNXT_FLAG_USING_MSIX)
8632                 pci_disable_msix(bp->pdev);
8633
8634         kfree(bp->irq_tbl);
8635         bp->irq_tbl = NULL;
8636         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8637 }
8638
8639 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8640 {
8641         int tcs = netdev_get_num_tc(bp->dev);
8642         bool irq_cleared = false;
8643         int rc;
8644
8645         if (!bnxt_need_reserve_rings(bp))
8646                 return 0;
8647
8648         if (irq_re_init && BNXT_NEW_RM(bp) &&
8649             bnxt_get_num_msix(bp) != bp->total_irqs) {
8650                 bnxt_ulp_irq_stop(bp);
8651                 bnxt_clear_int_mode(bp);
8652                 irq_cleared = true;
8653         }
8654         rc = __bnxt_reserve_rings(bp);
8655         if (irq_cleared) {
8656                 if (!rc)
8657                         rc = bnxt_init_int_mode(bp);
8658                 bnxt_ulp_irq_restart(bp, rc);
8659         }
8660         if (rc) {
8661                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8662                 return rc;
8663         }
8664         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8665                 netdev_err(bp->dev, "tx ring reservation failure\n");
8666                 netdev_reset_tc(bp->dev);
8667                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8668                 return -ENOMEM;
8669         }
8670         return 0;
8671 }
8672
8673 static void bnxt_free_irq(struct bnxt *bp)
8674 {
8675         struct bnxt_irq *irq;
8676         int i;
8677
8678 #ifdef CONFIG_RFS_ACCEL
8679         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8680         bp->dev->rx_cpu_rmap = NULL;
8681 #endif
8682         if (!bp->irq_tbl || !bp->bnapi)
8683                 return;
8684
8685         for (i = 0; i < bp->cp_nr_rings; i++) {
8686                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8687
8688                 irq = &bp->irq_tbl[map_idx];
8689                 if (irq->requested) {
8690                         if (irq->have_cpumask) {
8691                                 irq_set_affinity_hint(irq->vector, NULL);
8692                                 free_cpumask_var(irq->cpu_mask);
8693                                 irq->have_cpumask = 0;
8694                         }
8695                         free_irq(irq->vector, bp->bnapi[i]);
8696                 }
8697
8698                 irq->requested = 0;
8699         }
8700 }
8701
8702 static int bnxt_request_irq(struct bnxt *bp)
8703 {
8704         int i, j, rc = 0;
8705         unsigned long flags = 0;
8706 #ifdef CONFIG_RFS_ACCEL
8707         struct cpu_rmap *rmap;
8708 #endif
8709
8710         rc = bnxt_setup_int_mode(bp);
8711         if (rc) {
8712                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8713                            rc);
8714                 return rc;
8715         }
8716 #ifdef CONFIG_RFS_ACCEL
8717         rmap = bp->dev->rx_cpu_rmap;
8718 #endif
8719         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8720                 flags = IRQF_SHARED;
8721
8722         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8723                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8724                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8725
8726 #ifdef CONFIG_RFS_ACCEL
8727                 if (rmap && bp->bnapi[i]->rx_ring) {
8728                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8729                         if (rc)
8730                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8731                                             j);
8732                         j++;
8733                 }
8734 #endif
8735                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8736                                  bp->bnapi[i]);
8737                 if (rc)
8738                         break;
8739
8740                 irq->requested = 1;
8741
8742                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8743                         int numa_node = dev_to_node(&bp->pdev->dev);
8744
8745                         irq->have_cpumask = 1;
8746                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8747                                         irq->cpu_mask);
8748                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8749                         if (rc) {
8750                                 netdev_warn(bp->dev,
8751                                             "Set affinity failed, IRQ = %d\n",
8752                                             irq->vector);
8753                                 break;
8754                         }
8755                 }
8756         }
8757         return rc;
8758 }
8759
8760 static void bnxt_del_napi(struct bnxt *bp)
8761 {
8762         int i;
8763
8764         if (!bp->bnapi)
8765                 return;
8766
8767         for (i = 0; i < bp->cp_nr_rings; i++) {
8768                 struct bnxt_napi *bnapi = bp->bnapi[i];
8769
8770                 __netif_napi_del(&bnapi->napi);
8771         }
8772         /* We called __netif_napi_del(), we need
8773          * to respect an RCU grace period before freeing napi structures.
8774          */
8775         synchronize_net();
8776 }
8777
8778 static void bnxt_init_napi(struct bnxt *bp)
8779 {
8780         int i;
8781         unsigned int cp_nr_rings = bp->cp_nr_rings;
8782         struct bnxt_napi *bnapi;
8783
8784         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8785                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8786
8787                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8788                         poll_fn = bnxt_poll_p5;
8789                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8790                         cp_nr_rings--;
8791                 for (i = 0; i < cp_nr_rings; i++) {
8792                         bnapi = bp->bnapi[i];
8793                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8794                 }
8795                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8796                         bnapi = bp->bnapi[cp_nr_rings];
8797                         netif_napi_add(bp->dev, &bnapi->napi,
8798                                        bnxt_poll_nitroa0, 64);
8799                 }
8800         } else {
8801                 bnapi = bp->bnapi[0];
8802                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8803         }
8804 }
8805
8806 static void bnxt_disable_napi(struct bnxt *bp)
8807 {
8808         int i;
8809
8810         if (!bp->bnapi)
8811                 return;
8812
8813         for (i = 0; i < bp->cp_nr_rings; i++) {
8814                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8815
8816                 if (bp->bnapi[i]->rx_ring)
8817                         cancel_work_sync(&cpr->dim.work);
8818
8819                 napi_disable(&bp->bnapi[i]->napi);
8820         }
8821 }
8822
8823 static void bnxt_enable_napi(struct bnxt *bp)
8824 {
8825         int i;
8826
8827         for (i = 0; i < bp->cp_nr_rings; i++) {
8828                 struct bnxt_napi *bnapi = bp->bnapi[i];
8829                 struct bnxt_cp_ring_info *cpr;
8830
8831                 cpr = &bnapi->cp_ring;
8832                 if (bnapi->in_reset)
8833                         cpr->sw_stats.rx.rx_resets++;
8834                 bnapi->in_reset = false;
8835
8836                 if (bnapi->rx_ring) {
8837                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8838                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8839                 }
8840                 napi_enable(&bnapi->napi);
8841         }
8842 }
8843
8844 void bnxt_tx_disable(struct bnxt *bp)
8845 {
8846         int i;
8847         struct bnxt_tx_ring_info *txr;
8848
8849         if (bp->tx_ring) {
8850                 for (i = 0; i < bp->tx_nr_rings; i++) {
8851                         txr = &bp->tx_ring[i];
8852                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8853                 }
8854         }
8855         /* Stop all TX queues */
8856         netif_tx_disable(bp->dev);
8857         netif_carrier_off(bp->dev);
8858 }
8859
8860 void bnxt_tx_enable(struct bnxt *bp)
8861 {
8862         int i;
8863         struct bnxt_tx_ring_info *txr;
8864
8865         for (i = 0; i < bp->tx_nr_rings; i++) {
8866                 txr = &bp->tx_ring[i];
8867                 txr->dev_state = 0;
8868         }
8869         netif_tx_wake_all_queues(bp->dev);
8870         if (bp->link_info.link_up)
8871                 netif_carrier_on(bp->dev);
8872 }
8873
8874 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
8875 {
8876         u8 active_fec = link_info->active_fec_sig_mode &
8877                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
8878
8879         switch (active_fec) {
8880         default:
8881         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
8882                 return "None";
8883         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
8884                 return "Clause 74 BaseR";
8885         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
8886                 return "Clause 91 RS(528,514)";
8887         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
8888                 return "Clause 91 RS544_1XN";
8889         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
8890                 return "Clause 91 RS(544,514)";
8891         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
8892                 return "Clause 91 RS272_1XN";
8893         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
8894                 return "Clause 91 RS(272,257)";
8895         }
8896 }
8897
8898 static void bnxt_report_link(struct bnxt *bp)
8899 {
8900         if (bp->link_info.link_up) {
8901                 const char *duplex;
8902                 const char *flow_ctrl;
8903                 u32 speed;
8904                 u16 fec;
8905
8906                 netif_carrier_on(bp->dev);
8907                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8908                 if (speed == SPEED_UNKNOWN) {
8909                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
8910                         return;
8911                 }
8912                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8913                         duplex = "full";
8914                 else
8915                         duplex = "half";
8916                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8917                         flow_ctrl = "ON - receive & transmit";
8918                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8919                         flow_ctrl = "ON - transmit";
8920                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8921                         flow_ctrl = "ON - receive";
8922                 else
8923                         flow_ctrl = "none";
8924                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8925                             speed, duplex, flow_ctrl);
8926                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8927                         netdev_info(bp->dev, "EEE is %s\n",
8928                                     bp->eee.eee_active ? "active" :
8929                                                          "not active");
8930                 fec = bp->link_info.fec_cfg;
8931                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8932                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
8933                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8934                                     bnxt_report_fec(&bp->link_info));
8935         } else {
8936                 netif_carrier_off(bp->dev);
8937                 netdev_err(bp->dev, "NIC Link is Down\n");
8938         }
8939 }
8940
8941 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
8942 {
8943         if (!resp->supported_speeds_auto_mode &&
8944             !resp->supported_speeds_force_mode &&
8945             !resp->supported_pam4_speeds_auto_mode &&
8946             !resp->supported_pam4_speeds_force_mode)
8947                 return true;
8948         return false;
8949 }
8950
8951 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8952 {
8953         int rc = 0;
8954         struct hwrm_port_phy_qcaps_input req = {0};
8955         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8956         struct bnxt_link_info *link_info = &bp->link_info;
8957
8958         bp->flags &= ~BNXT_FLAG_EEE_CAP;
8959         if (bp->test_info)
8960                 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8961                                           BNXT_TEST_FL_AN_PHY_LPBK);
8962         if (bp->hwrm_spec_code < 0x10201)
8963                 return 0;
8964
8965         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8966
8967         mutex_lock(&bp->hwrm_cmd_lock);
8968         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8969         if (rc)
8970                 goto hwrm_phy_qcaps_exit;
8971
8972         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8973                 struct ethtool_eee *eee = &bp->eee;
8974                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8975
8976                 bp->flags |= BNXT_FLAG_EEE_CAP;
8977                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8978                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8979                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8980                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8981                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8982         }
8983         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8984                 if (bp->test_info)
8985                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8986         }
8987         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8988                 if (bp->test_info)
8989                         bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8990         }
8991         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8992                 if (BNXT_PF(bp))
8993                         bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8994         }
8995         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8996                 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8997
8998         if (bp->hwrm_spec_code >= 0x10a01) {
8999                 if (bnxt_phy_qcaps_no_speed(resp)) {
9000                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9001                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9002                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9003                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9004                         netdev_info(bp->dev, "Ethernet link enabled\n");
9005                         /* Phy re-enabled, reprobe the speeds */
9006                         link_info->support_auto_speeds = 0;
9007                         link_info->support_pam4_auto_speeds = 0;
9008                 }
9009         }
9010         if (resp->supported_speeds_auto_mode)
9011                 link_info->support_auto_speeds =
9012                         le16_to_cpu(resp->supported_speeds_auto_mode);
9013         if (resp->supported_pam4_speeds_auto_mode)
9014                 link_info->support_pam4_auto_speeds =
9015                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9016
9017         bp->port_count = resp->port_cnt;
9018
9019 hwrm_phy_qcaps_exit:
9020         mutex_unlock(&bp->hwrm_cmd_lock);
9021         return rc;
9022 }
9023
9024 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9025 {
9026         u16 diff = advertising ^ supported;
9027
9028         return ((supported | diff) != supported);
9029 }
9030
9031 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9032 {
9033         int rc = 0;
9034         struct bnxt_link_info *link_info = &bp->link_info;
9035         struct hwrm_port_phy_qcfg_input req = {0};
9036         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9037         u8 link_up = link_info->link_up;
9038         bool support_changed = false;
9039
9040         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9041
9042         mutex_lock(&bp->hwrm_cmd_lock);
9043         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9044         if (rc) {
9045                 mutex_unlock(&bp->hwrm_cmd_lock);
9046                 return rc;
9047         }
9048
9049         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9050         link_info->phy_link_status = resp->link;
9051         link_info->duplex = resp->duplex_cfg;
9052         if (bp->hwrm_spec_code >= 0x10800)
9053                 link_info->duplex = resp->duplex_state;
9054         link_info->pause = resp->pause;
9055         link_info->auto_mode = resp->auto_mode;
9056         link_info->auto_pause_setting = resp->auto_pause;
9057         link_info->lp_pause = resp->link_partner_adv_pause;
9058         link_info->force_pause_setting = resp->force_pause;
9059         link_info->duplex_setting = resp->duplex_cfg;
9060         if (link_info->phy_link_status == BNXT_LINK_LINK)
9061                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9062         else
9063                 link_info->link_speed = 0;
9064         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9065         link_info->force_pam4_link_speed =
9066                 le16_to_cpu(resp->force_pam4_link_speed);
9067         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9068         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9069         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9070         link_info->auto_pam4_link_speeds =
9071                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9072         link_info->lp_auto_link_speeds =
9073                 le16_to_cpu(resp->link_partner_adv_speeds);
9074         link_info->lp_auto_pam4_link_speeds =
9075                 resp->link_partner_pam4_adv_speeds;
9076         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9077         link_info->phy_ver[0] = resp->phy_maj;
9078         link_info->phy_ver[1] = resp->phy_min;
9079         link_info->phy_ver[2] = resp->phy_bld;
9080         link_info->media_type = resp->media_type;
9081         link_info->phy_type = resp->phy_type;
9082         link_info->transceiver = resp->xcvr_pkg_type;
9083         link_info->phy_addr = resp->eee_config_phy_addr &
9084                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9085         link_info->module_status = resp->module_status;
9086
9087         if (bp->flags & BNXT_FLAG_EEE_CAP) {
9088                 struct ethtool_eee *eee = &bp->eee;
9089                 u16 fw_speeds;
9090
9091                 eee->eee_active = 0;
9092                 if (resp->eee_config_phy_addr &
9093                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9094                         eee->eee_active = 1;
9095                         fw_speeds = le16_to_cpu(
9096                                 resp->link_partner_adv_eee_link_speed_mask);
9097                         eee->lp_advertised =
9098                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9099                 }
9100
9101                 /* Pull initial EEE config */
9102                 if (!chng_link_state) {
9103                         if (resp->eee_config_phy_addr &
9104                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9105                                 eee->eee_enabled = 1;
9106
9107                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9108                         eee->advertised =
9109                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9110
9111                         if (resp->eee_config_phy_addr &
9112                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9113                                 __le32 tmr;
9114
9115                                 eee->tx_lpi_enabled = 1;
9116                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9117                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9118                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9119                         }
9120                 }
9121         }
9122
9123         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9124         if (bp->hwrm_spec_code >= 0x10504) {
9125                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9126                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9127         }
9128         /* TODO: need to add more logic to report VF link */
9129         if (chng_link_state) {
9130                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9131                         link_info->link_up = 1;
9132                 else
9133                         link_info->link_up = 0;
9134                 if (link_up != link_info->link_up)
9135                         bnxt_report_link(bp);
9136         } else {
9137                 /* alwasy link down if not require to update link state */
9138                 link_info->link_up = 0;
9139         }
9140         mutex_unlock(&bp->hwrm_cmd_lock);
9141
9142         if (!BNXT_PHY_CFG_ABLE(bp))
9143                 return 0;
9144
9145         /* Check if any advertised speeds are no longer supported. The caller
9146          * holds the link_lock mutex, so we can modify link_info settings.
9147          */
9148         if (bnxt_support_dropped(link_info->advertising,
9149                                  link_info->support_auto_speeds)) {
9150                 link_info->advertising = link_info->support_auto_speeds;
9151                 support_changed = true;
9152         }
9153         if (bnxt_support_dropped(link_info->advertising_pam4,
9154                                  link_info->support_pam4_auto_speeds)) {
9155                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9156                 support_changed = true;
9157         }
9158         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9159                 bnxt_hwrm_set_link_setting(bp, true, false);
9160         return 0;
9161 }
9162
9163 static void bnxt_get_port_module_status(struct bnxt *bp)
9164 {
9165         struct bnxt_link_info *link_info = &bp->link_info;
9166         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9167         u8 module_status;
9168
9169         if (bnxt_update_link(bp, true))
9170                 return;
9171
9172         module_status = link_info->module_status;
9173         switch (module_status) {
9174         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9175         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9176         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9177                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9178                             bp->pf.port_id);
9179                 if (bp->hwrm_spec_code >= 0x10201) {
9180                         netdev_warn(bp->dev, "Module part number %s\n",
9181                                     resp->phy_vendor_partnumber);
9182                 }
9183                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9184                         netdev_warn(bp->dev, "TX is disabled\n");
9185                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9186                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9187         }
9188 }
9189
9190 static void
9191 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9192 {
9193         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9194                 if (bp->hwrm_spec_code >= 0x10201)
9195                         req->auto_pause =
9196                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9197                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9198                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9199                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9200                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9201                 req->enables |=
9202                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9203         } else {
9204                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9205                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9206                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9207                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9208                 req->enables |=
9209                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9210                 if (bp->hwrm_spec_code >= 0x10201) {
9211                         req->auto_pause = req->force_pause;
9212                         req->enables |= cpu_to_le32(
9213                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9214                 }
9215         }
9216 }
9217
9218 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9219 {
9220         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9221                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9222                 if (bp->link_info.advertising) {
9223                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9224                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9225                 }
9226                 if (bp->link_info.advertising_pam4) {
9227                         req->enables |=
9228                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9229                         req->auto_link_pam4_speed_mask =
9230                                 cpu_to_le16(bp->link_info.advertising_pam4);
9231                 }
9232                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9233                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9234         } else {
9235                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9236                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9237                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9238                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9239                 } else {
9240                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9241                 }
9242         }
9243
9244         /* tell chimp that the setting takes effect immediately */
9245         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9246 }
9247
9248 int bnxt_hwrm_set_pause(struct bnxt *bp)
9249 {
9250         struct hwrm_port_phy_cfg_input req = {0};
9251         int rc;
9252
9253         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9254         bnxt_hwrm_set_pause_common(bp, &req);
9255
9256         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9257             bp->link_info.force_link_chng)
9258                 bnxt_hwrm_set_link_common(bp, &req);
9259
9260         mutex_lock(&bp->hwrm_cmd_lock);
9261         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9262         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9263                 /* since changing of pause setting doesn't trigger any link
9264                  * change event, the driver needs to update the current pause
9265                  * result upon successfully return of the phy_cfg command
9266                  */
9267                 bp->link_info.pause =
9268                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9269                 bp->link_info.auto_pause_setting = 0;
9270                 if (!bp->link_info.force_link_chng)
9271                         bnxt_report_link(bp);
9272         }
9273         bp->link_info.force_link_chng = false;
9274         mutex_unlock(&bp->hwrm_cmd_lock);
9275         return rc;
9276 }
9277
9278 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9279                               struct hwrm_port_phy_cfg_input *req)
9280 {
9281         struct ethtool_eee *eee = &bp->eee;
9282
9283         if (eee->eee_enabled) {
9284                 u16 eee_speeds;
9285                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9286
9287                 if (eee->tx_lpi_enabled)
9288                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9289                 else
9290                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9291
9292                 req->flags |= cpu_to_le32(flags);
9293                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9294                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9295                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9296         } else {
9297                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9298         }
9299 }
9300
9301 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9302 {
9303         struct hwrm_port_phy_cfg_input req = {0};
9304
9305         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9306         if (set_pause)
9307                 bnxt_hwrm_set_pause_common(bp, &req);
9308
9309         bnxt_hwrm_set_link_common(bp, &req);
9310
9311         if (set_eee)
9312                 bnxt_hwrm_set_eee(bp, &req);
9313         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9314 }
9315
9316 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9317 {
9318         struct hwrm_port_phy_cfg_input req = {0};
9319
9320         if (!BNXT_SINGLE_PF(bp))
9321                 return 0;
9322
9323         if (pci_num_vf(bp->pdev))
9324                 return 0;
9325
9326         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9327         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9328         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9329 }
9330
9331 static int bnxt_fw_init_one(struct bnxt *bp);
9332
9333 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9334 {
9335         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9336         struct hwrm_func_drv_if_change_input req = {0};
9337         bool resc_reinit = false, fw_reset = false;
9338         u32 flags = 0;
9339         int rc;
9340
9341         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9342                 return 0;
9343
9344         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9345         if (up)
9346                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9347         mutex_lock(&bp->hwrm_cmd_lock);
9348         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9349         if (!rc)
9350                 flags = le32_to_cpu(resp->flags);
9351         mutex_unlock(&bp->hwrm_cmd_lock);
9352         if (rc)
9353                 return rc;
9354
9355         if (!up)
9356                 return 0;
9357
9358         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9359                 resc_reinit = true;
9360         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9361                 fw_reset = true;
9362
9363         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9364                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9365                 return -ENODEV;
9366         }
9367         if (resc_reinit || fw_reset) {
9368                 if (fw_reset) {
9369                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9370                                 bnxt_ulp_stop(bp);
9371                         bnxt_free_ctx_mem(bp);
9372                         kfree(bp->ctx);
9373                         bp->ctx = NULL;
9374                         bnxt_dcb_free(bp);
9375                         rc = bnxt_fw_init_one(bp);
9376                         if (rc) {
9377                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9378                                 return rc;
9379                         }
9380                         bnxt_clear_int_mode(bp);
9381                         rc = bnxt_init_int_mode(bp);
9382                         if (rc) {
9383                                 netdev_err(bp->dev, "init int mode failed\n");
9384                                 return rc;
9385                         }
9386                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9387                 }
9388                 if (BNXT_NEW_RM(bp)) {
9389                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9390
9391                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9392                         hw_resc->resv_cp_rings = 0;
9393                         hw_resc->resv_stat_ctxs = 0;
9394                         hw_resc->resv_irqs = 0;
9395                         hw_resc->resv_tx_rings = 0;
9396                         hw_resc->resv_rx_rings = 0;
9397                         hw_resc->resv_hw_ring_grps = 0;
9398                         hw_resc->resv_vnics = 0;
9399                         if (!fw_reset) {
9400                                 bp->tx_nr_rings = 0;
9401                                 bp->rx_nr_rings = 0;
9402                         }
9403                 }
9404         }
9405         return 0;
9406 }
9407
9408 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9409 {
9410         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9411         struct hwrm_port_led_qcaps_input req = {0};
9412         struct bnxt_pf_info *pf = &bp->pf;
9413         int rc;
9414
9415         bp->num_leds = 0;
9416         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9417                 return 0;
9418
9419         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9420         req.port_id = cpu_to_le16(pf->port_id);
9421         mutex_lock(&bp->hwrm_cmd_lock);
9422         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9423         if (rc) {
9424                 mutex_unlock(&bp->hwrm_cmd_lock);
9425                 return rc;
9426         }
9427         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9428                 int i;
9429
9430                 bp->num_leds = resp->num_leds;
9431                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9432                                                  bp->num_leds);
9433                 for (i = 0; i < bp->num_leds; i++) {
9434                         struct bnxt_led_info *led = &bp->leds[i];
9435                         __le16 caps = led->led_state_caps;
9436
9437                         if (!led->led_group_id ||
9438                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9439                                 bp->num_leds = 0;
9440                                 break;
9441                         }
9442                 }
9443         }
9444         mutex_unlock(&bp->hwrm_cmd_lock);
9445         return 0;
9446 }
9447
9448 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9449 {
9450         struct hwrm_wol_filter_alloc_input req = {0};
9451         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9452         int rc;
9453
9454         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9455         req.port_id = cpu_to_le16(bp->pf.port_id);
9456         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9457         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9458         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9459         mutex_lock(&bp->hwrm_cmd_lock);
9460         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9461         if (!rc)
9462                 bp->wol_filter_id = resp->wol_filter_id;
9463         mutex_unlock(&bp->hwrm_cmd_lock);
9464         return rc;
9465 }
9466
9467 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9468 {
9469         struct hwrm_wol_filter_free_input req = {0};
9470
9471         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9472         req.port_id = cpu_to_le16(bp->pf.port_id);
9473         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9474         req.wol_filter_id = bp->wol_filter_id;
9475         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9476 }
9477
9478 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9479 {
9480         struct hwrm_wol_filter_qcfg_input req = {0};
9481         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9482         u16 next_handle = 0;
9483         int rc;
9484
9485         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9486         req.port_id = cpu_to_le16(bp->pf.port_id);
9487         req.handle = cpu_to_le16(handle);
9488         mutex_lock(&bp->hwrm_cmd_lock);
9489         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9490         if (!rc) {
9491                 next_handle = le16_to_cpu(resp->next_handle);
9492                 if (next_handle != 0) {
9493                         if (resp->wol_type ==
9494                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9495                                 bp->wol = 1;
9496                                 bp->wol_filter_id = resp->wol_filter_id;
9497                         }
9498                 }
9499         }
9500         mutex_unlock(&bp->hwrm_cmd_lock);
9501         return next_handle;
9502 }
9503
9504 static void bnxt_get_wol_settings(struct bnxt *bp)
9505 {
9506         u16 handle = 0;
9507
9508         bp->wol = 0;
9509         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9510                 return;
9511
9512         do {
9513                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9514         } while (handle && handle != 0xffff);
9515 }
9516
9517 #ifdef CONFIG_BNXT_HWMON
9518 static ssize_t bnxt_show_temp(struct device *dev,
9519                               struct device_attribute *devattr, char *buf)
9520 {
9521         struct hwrm_temp_monitor_query_input req = {0};
9522         struct hwrm_temp_monitor_query_output *resp;
9523         struct bnxt *bp = dev_get_drvdata(dev);
9524         u32 len = 0;
9525         int rc;
9526
9527         resp = bp->hwrm_cmd_resp_addr;
9528         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9529         mutex_lock(&bp->hwrm_cmd_lock);
9530         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9531         if (!rc)
9532                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9533         mutex_unlock(&bp->hwrm_cmd_lock);
9534         return rc ?: len;
9535 }
9536 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9537
9538 static struct attribute *bnxt_attrs[] = {
9539         &sensor_dev_attr_temp1_input.dev_attr.attr,
9540         NULL
9541 };
9542 ATTRIBUTE_GROUPS(bnxt);
9543
9544 static void bnxt_hwmon_close(struct bnxt *bp)
9545 {
9546         if (bp->hwmon_dev) {
9547                 hwmon_device_unregister(bp->hwmon_dev);
9548                 bp->hwmon_dev = NULL;
9549         }
9550 }
9551
9552 static void bnxt_hwmon_open(struct bnxt *bp)
9553 {
9554         struct hwrm_temp_monitor_query_input req = {0};
9555         struct pci_dev *pdev = bp->pdev;
9556         int rc;
9557
9558         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9559         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9560         if (rc == -EACCES || rc == -EOPNOTSUPP) {
9561                 bnxt_hwmon_close(bp);
9562                 return;
9563         }
9564
9565         if (bp->hwmon_dev)
9566                 return;
9567
9568         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9569                                                           DRV_MODULE_NAME, bp,
9570                                                           bnxt_groups);
9571         if (IS_ERR(bp->hwmon_dev)) {
9572                 bp->hwmon_dev = NULL;
9573                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9574         }
9575 }
9576 #else
9577 static void bnxt_hwmon_close(struct bnxt *bp)
9578 {
9579 }
9580
9581 static void bnxt_hwmon_open(struct bnxt *bp)
9582 {
9583 }
9584 #endif
9585
9586 static bool bnxt_eee_config_ok(struct bnxt *bp)
9587 {
9588         struct ethtool_eee *eee = &bp->eee;
9589         struct bnxt_link_info *link_info = &bp->link_info;
9590
9591         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9592                 return true;
9593
9594         if (eee->eee_enabled) {
9595                 u32 advertising =
9596                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9597
9598                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9599                         eee->eee_enabled = 0;
9600                         return false;
9601                 }
9602                 if (eee->advertised & ~advertising) {
9603                         eee->advertised = advertising & eee->supported;
9604                         return false;
9605                 }
9606         }
9607         return true;
9608 }
9609
9610 static int bnxt_update_phy_setting(struct bnxt *bp)
9611 {
9612         int rc;
9613         bool update_link = false;
9614         bool update_pause = false;
9615         bool update_eee = false;
9616         struct bnxt_link_info *link_info = &bp->link_info;
9617
9618         rc = bnxt_update_link(bp, true);
9619         if (rc) {
9620                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9621                            rc);
9622                 return rc;
9623         }
9624         if (!BNXT_SINGLE_PF(bp))
9625                 return 0;
9626
9627         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9628             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9629             link_info->req_flow_ctrl)
9630                 update_pause = true;
9631         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9632             link_info->force_pause_setting != link_info->req_flow_ctrl)
9633                 update_pause = true;
9634         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9635                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9636                         update_link = true;
9637                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9638                     link_info->req_link_speed != link_info->force_link_speed)
9639                         update_link = true;
9640                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9641                          link_info->req_link_speed != link_info->force_pam4_link_speed)
9642                         update_link = true;
9643                 if (link_info->req_duplex != link_info->duplex_setting)
9644                         update_link = true;
9645         } else {
9646                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9647                         update_link = true;
9648                 if (link_info->advertising != link_info->auto_link_speeds ||
9649                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9650                         update_link = true;
9651         }
9652
9653         /* The last close may have shutdown the link, so need to call
9654          * PHY_CFG to bring it back up.
9655          */
9656         if (!bp->link_info.link_up)
9657                 update_link = true;
9658
9659         if (!bnxt_eee_config_ok(bp))
9660                 update_eee = true;
9661
9662         if (update_link)
9663                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9664         else if (update_pause)
9665                 rc = bnxt_hwrm_set_pause(bp);
9666         if (rc) {
9667                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9668                            rc);
9669                 return rc;
9670         }
9671
9672         return rc;
9673 }
9674
9675 /* Common routine to pre-map certain register block to different GRC window.
9676  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9677  * in PF and 3 windows in VF that can be customized to map in different
9678  * register blocks.
9679  */
9680 static void bnxt_preset_reg_win(struct bnxt *bp)
9681 {
9682         if (BNXT_PF(bp)) {
9683                 /* CAG registers map to GRC window #4 */
9684                 writel(BNXT_CAG_REG_BASE,
9685                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9686         }
9687 }
9688
9689 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9690
9691 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9692 {
9693         int rc = 0;
9694
9695         bnxt_preset_reg_win(bp);
9696         netif_carrier_off(bp->dev);
9697         if (irq_re_init) {
9698                 /* Reserve rings now if none were reserved at driver probe. */
9699                 rc = bnxt_init_dflt_ring_mode(bp);
9700                 if (rc) {
9701                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9702                         return rc;
9703                 }
9704         }
9705         rc = bnxt_reserve_rings(bp, irq_re_init);
9706         if (rc)
9707                 return rc;
9708         if ((bp->flags & BNXT_FLAG_RFS) &&
9709             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9710                 /* disable RFS if falling back to INTA */
9711                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9712                 bp->flags &= ~BNXT_FLAG_RFS;
9713         }
9714
9715         rc = bnxt_alloc_mem(bp, irq_re_init);
9716         if (rc) {
9717                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9718                 goto open_err_free_mem;
9719         }
9720
9721         if (irq_re_init) {
9722                 bnxt_init_napi(bp);
9723                 rc = bnxt_request_irq(bp);
9724                 if (rc) {
9725                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9726                         goto open_err_irq;
9727                 }
9728         }
9729
9730         rc = bnxt_init_nic(bp, irq_re_init);
9731         if (rc) {
9732                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9733                 goto open_err_irq;
9734         }
9735
9736         bnxt_enable_napi(bp);
9737         bnxt_debug_dev_init(bp);
9738
9739         if (link_re_init) {
9740                 mutex_lock(&bp->link_lock);
9741                 rc = bnxt_update_phy_setting(bp);
9742                 mutex_unlock(&bp->link_lock);
9743                 if (rc) {
9744                         netdev_warn(bp->dev, "failed to update phy settings\n");
9745                         if (BNXT_SINGLE_PF(bp)) {
9746                                 bp->link_info.phy_retry = true;
9747                                 bp->link_info.phy_retry_expires =
9748                                         jiffies + 5 * HZ;
9749                         }
9750                 }
9751         }
9752
9753         if (irq_re_init)
9754                 udp_tunnel_nic_reset_ntf(bp->dev);
9755
9756         set_bit(BNXT_STATE_OPEN, &bp->state);
9757         bnxt_enable_int(bp);
9758         /* Enable TX queues */
9759         bnxt_tx_enable(bp);
9760         mod_timer(&bp->timer, jiffies + bp->current_interval);
9761         /* Poll link status and check for SFP+ module status */
9762         bnxt_get_port_module_status(bp);
9763
9764         /* VF-reps may need to be re-opened after the PF is re-opened */
9765         if (BNXT_PF(bp))
9766                 bnxt_vf_reps_open(bp);
9767         return 0;
9768
9769 open_err_irq:
9770         bnxt_del_napi(bp);
9771
9772 open_err_free_mem:
9773         bnxt_free_skbs(bp);
9774         bnxt_free_irq(bp);
9775         bnxt_free_mem(bp, true);
9776         return rc;
9777 }
9778
9779 /* rtnl_lock held */
9780 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9781 {
9782         int rc = 0;
9783
9784         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
9785                 rc = -EIO;
9786         if (!rc)
9787                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9788         if (rc) {
9789                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9790                 dev_close(bp->dev);
9791         }
9792         return rc;
9793 }
9794
9795 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9796  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9797  * self tests.
9798  */
9799 int bnxt_half_open_nic(struct bnxt *bp)
9800 {
9801         int rc = 0;
9802
9803         rc = bnxt_alloc_mem(bp, false);
9804         if (rc) {
9805                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9806                 goto half_open_err;
9807         }
9808         rc = bnxt_init_nic(bp, false);
9809         if (rc) {
9810                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9811                 goto half_open_err;
9812         }
9813         return 0;
9814
9815 half_open_err:
9816         bnxt_free_skbs(bp);
9817         bnxt_free_mem(bp, false);
9818         dev_close(bp->dev);
9819         return rc;
9820 }
9821
9822 /* rtnl_lock held, this call can only be made after a previous successful
9823  * call to bnxt_half_open_nic().
9824  */
9825 void bnxt_half_close_nic(struct bnxt *bp)
9826 {
9827         bnxt_hwrm_resource_free(bp, false, false);
9828         bnxt_free_skbs(bp);
9829         bnxt_free_mem(bp, false);
9830 }
9831
9832 static void bnxt_reenable_sriov(struct bnxt *bp)
9833 {
9834         if (BNXT_PF(bp)) {
9835                 struct bnxt_pf_info *pf = &bp->pf;
9836                 int n = pf->active_vfs;
9837
9838                 if (n)
9839                         bnxt_cfg_hw_sriov(bp, &n, true);
9840         }
9841 }
9842
9843 static int bnxt_open(struct net_device *dev)
9844 {
9845         struct bnxt *bp = netdev_priv(dev);
9846         int rc;
9847
9848         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9849                 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9850                 return -ENODEV;
9851         }
9852
9853         rc = bnxt_hwrm_if_change(bp, true);
9854         if (rc)
9855                 return rc;
9856         rc = __bnxt_open_nic(bp, true, true);
9857         if (rc) {
9858                 bnxt_hwrm_if_change(bp, false);
9859         } else {
9860                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9861                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9862                                 bnxt_ulp_start(bp, 0);
9863                                 bnxt_reenable_sriov(bp);
9864                         }
9865                 }
9866                 bnxt_hwmon_open(bp);
9867         }
9868
9869         return rc;
9870 }
9871
9872 static bool bnxt_drv_busy(struct bnxt *bp)
9873 {
9874         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9875                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9876 }
9877
9878 static void bnxt_get_ring_stats(struct bnxt *bp,
9879                                 struct rtnl_link_stats64 *stats);
9880
9881 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9882                              bool link_re_init)
9883 {
9884         /* Close the VF-reps before closing PF */
9885         if (BNXT_PF(bp))
9886                 bnxt_vf_reps_close(bp);
9887
9888         /* Change device state to avoid TX queue wake up's */
9889         bnxt_tx_disable(bp);
9890
9891         clear_bit(BNXT_STATE_OPEN, &bp->state);
9892         smp_mb__after_atomic();
9893         while (bnxt_drv_busy(bp))
9894                 msleep(20);
9895
9896         /* Flush rings and and disable interrupts */
9897         bnxt_shutdown_nic(bp, irq_re_init);
9898
9899         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9900
9901         bnxt_debug_dev_exit(bp);
9902         bnxt_disable_napi(bp);
9903         del_timer_sync(&bp->timer);
9904         bnxt_free_skbs(bp);
9905
9906         /* Save ring stats before shutdown */
9907         if (bp->bnapi && irq_re_init)
9908                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9909         if (irq_re_init) {
9910                 bnxt_free_irq(bp);
9911                 bnxt_del_napi(bp);
9912         }
9913         bnxt_free_mem(bp, irq_re_init);
9914 }
9915
9916 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9917 {
9918         int rc = 0;
9919
9920         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9921                 /* If we get here, it means firmware reset is in progress
9922                  * while we are trying to close.  We can safely proceed with
9923                  * the close because we are holding rtnl_lock().  Some firmware
9924                  * messages may fail as we proceed to close.  We set the
9925                  * ABORT_ERR flag here so that the FW reset thread will later
9926                  * abort when it gets the rtnl_lock() and sees the flag.
9927                  */
9928                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9929                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9930         }
9931
9932 #ifdef CONFIG_BNXT_SRIOV
9933         if (bp->sriov_cfg) {
9934                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9935                                                       !bp->sriov_cfg,
9936                                                       BNXT_SRIOV_CFG_WAIT_TMO);
9937                 if (rc)
9938                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9939         }
9940 #endif
9941         __bnxt_close_nic(bp, irq_re_init, link_re_init);
9942         return rc;
9943 }
9944
9945 static int bnxt_close(struct net_device *dev)
9946 {
9947         struct bnxt *bp = netdev_priv(dev);
9948
9949         bnxt_hwmon_close(bp);
9950         bnxt_close_nic(bp, true, true);
9951         bnxt_hwrm_shutdown_link(bp);
9952         bnxt_hwrm_if_change(bp, false);
9953         return 0;
9954 }
9955
9956 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9957                                    u16 *val)
9958 {
9959         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9960         struct hwrm_port_phy_mdio_read_input req = {0};
9961         int rc;
9962
9963         if (bp->hwrm_spec_code < 0x10a00)
9964                 return -EOPNOTSUPP;
9965
9966         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9967         req.port_id = cpu_to_le16(bp->pf.port_id);
9968         req.phy_addr = phy_addr;
9969         req.reg_addr = cpu_to_le16(reg & 0x1f);
9970         if (mdio_phy_id_is_c45(phy_addr)) {
9971                 req.cl45_mdio = 1;
9972                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9973                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9974                 req.reg_addr = cpu_to_le16(reg);
9975         }
9976
9977         mutex_lock(&bp->hwrm_cmd_lock);
9978         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9979         if (!rc)
9980                 *val = le16_to_cpu(resp->reg_data);
9981         mutex_unlock(&bp->hwrm_cmd_lock);
9982         return rc;
9983 }
9984
9985 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9986                                     u16 val)
9987 {
9988         struct hwrm_port_phy_mdio_write_input req = {0};
9989
9990         if (bp->hwrm_spec_code < 0x10a00)
9991                 return -EOPNOTSUPP;
9992
9993         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9994         req.port_id = cpu_to_le16(bp->pf.port_id);
9995         req.phy_addr = phy_addr;
9996         req.reg_addr = cpu_to_le16(reg & 0x1f);
9997         if (mdio_phy_id_is_c45(phy_addr)) {
9998                 req.cl45_mdio = 1;
9999                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10000                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10001                 req.reg_addr = cpu_to_le16(reg);
10002         }
10003         req.reg_data = cpu_to_le16(val);
10004
10005         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10006 }
10007
10008 /* rtnl_lock held */
10009 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10010 {
10011         struct mii_ioctl_data *mdio = if_mii(ifr);
10012         struct bnxt *bp = netdev_priv(dev);
10013         int rc;
10014
10015         switch (cmd) {
10016         case SIOCGMIIPHY:
10017                 mdio->phy_id = bp->link_info.phy_addr;
10018
10019                 fallthrough;
10020         case SIOCGMIIREG: {
10021                 u16 mii_regval = 0;
10022
10023                 if (!netif_running(dev))
10024                         return -EAGAIN;
10025
10026                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10027                                              &mii_regval);
10028                 mdio->val_out = mii_regval;
10029                 return rc;
10030         }
10031
10032         case SIOCSMIIREG:
10033                 if (!netif_running(dev))
10034                         return -EAGAIN;
10035
10036                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10037                                                 mdio->val_in);
10038
10039         default:
10040                 /* do nothing */
10041                 break;
10042         }
10043         return -EOPNOTSUPP;
10044 }
10045
10046 static void bnxt_get_ring_stats(struct bnxt *bp,
10047                                 struct rtnl_link_stats64 *stats)
10048 {
10049         int i;
10050
10051         for (i = 0; i < bp->cp_nr_rings; i++) {
10052                 struct bnxt_napi *bnapi = bp->bnapi[i];
10053                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10054                 u64 *sw = cpr->stats.sw_stats;
10055
10056                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10057                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10058                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10059
10060                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10061                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10062                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10063
10064                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10065                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10066                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10067
10068                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10069                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10070                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10071
10072                 stats->rx_missed_errors +=
10073                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10074
10075                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10076
10077                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10078         }
10079 }
10080
10081 static void bnxt_add_prev_stats(struct bnxt *bp,
10082                                 struct rtnl_link_stats64 *stats)
10083 {
10084         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10085
10086         stats->rx_packets += prev_stats->rx_packets;
10087         stats->tx_packets += prev_stats->tx_packets;
10088         stats->rx_bytes += prev_stats->rx_bytes;
10089         stats->tx_bytes += prev_stats->tx_bytes;
10090         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10091         stats->multicast += prev_stats->multicast;
10092         stats->tx_dropped += prev_stats->tx_dropped;
10093 }
10094
10095 static void
10096 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10097 {
10098         struct bnxt *bp = netdev_priv(dev);
10099
10100         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10101         /* Make sure bnxt_close_nic() sees that we are reading stats before
10102          * we check the BNXT_STATE_OPEN flag.
10103          */
10104         smp_mb__after_atomic();
10105         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10106                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10107                 *stats = bp->net_stats_prev;
10108                 return;
10109         }
10110
10111         bnxt_get_ring_stats(bp, stats);
10112         bnxt_add_prev_stats(bp, stats);
10113
10114         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10115                 u64 *rx = bp->port_stats.sw_stats;
10116                 u64 *tx = bp->port_stats.sw_stats +
10117                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10118
10119                 stats->rx_crc_errors =
10120                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10121                 stats->rx_frame_errors =
10122                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10123                 stats->rx_length_errors =
10124                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10125                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10126                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10127                 stats->rx_errors =
10128                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10129                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10130                 stats->collisions =
10131                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10132                 stats->tx_fifo_errors =
10133                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10134                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10135         }
10136         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10137 }
10138
10139 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10140 {
10141         struct net_device *dev = bp->dev;
10142         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10143         struct netdev_hw_addr *ha;
10144         u8 *haddr;
10145         int mc_count = 0;
10146         bool update = false;
10147         int off = 0;
10148
10149         netdev_for_each_mc_addr(ha, dev) {
10150                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10151                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10152                         vnic->mc_list_count = 0;
10153                         return false;
10154                 }
10155                 haddr = ha->addr;
10156                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10157                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10158                         update = true;
10159                 }
10160                 off += ETH_ALEN;
10161                 mc_count++;
10162         }
10163         if (mc_count)
10164                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10165
10166         if (mc_count != vnic->mc_list_count) {
10167                 vnic->mc_list_count = mc_count;
10168                 update = true;
10169         }
10170         return update;
10171 }
10172
10173 static bool bnxt_uc_list_updated(struct bnxt *bp)
10174 {
10175         struct net_device *dev = bp->dev;
10176         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10177         struct netdev_hw_addr *ha;
10178         int off = 0;
10179
10180         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10181                 return true;
10182
10183         netdev_for_each_uc_addr(ha, dev) {
10184                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10185                         return true;
10186
10187                 off += ETH_ALEN;
10188         }
10189         return false;
10190 }
10191
10192 static void bnxt_set_rx_mode(struct net_device *dev)
10193 {
10194         struct bnxt *bp = netdev_priv(dev);
10195         struct bnxt_vnic_info *vnic;
10196         bool mc_update = false;
10197         bool uc_update;
10198         u32 mask;
10199
10200         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10201                 return;
10202
10203         vnic = &bp->vnic_info[0];
10204         mask = vnic->rx_mask;
10205         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10206                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10207                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10208                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10209
10210         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10211                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10212
10213         uc_update = bnxt_uc_list_updated(bp);
10214
10215         if (dev->flags & IFF_BROADCAST)
10216                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10217         if (dev->flags & IFF_ALLMULTI) {
10218                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10219                 vnic->mc_list_count = 0;
10220         } else {
10221                 mc_update = bnxt_mc_list_updated(bp, &mask);
10222         }
10223
10224         if (mask != vnic->rx_mask || uc_update || mc_update) {
10225                 vnic->rx_mask = mask;
10226
10227                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10228                 bnxt_queue_sp_work(bp);
10229         }
10230 }
10231
10232 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10233 {
10234         struct net_device *dev = bp->dev;
10235         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10236         struct netdev_hw_addr *ha;
10237         int i, off = 0, rc;
10238         bool uc_update;
10239
10240         netif_addr_lock_bh(dev);
10241         uc_update = bnxt_uc_list_updated(bp);
10242         netif_addr_unlock_bh(dev);
10243
10244         if (!uc_update)
10245                 goto skip_uc;
10246
10247         mutex_lock(&bp->hwrm_cmd_lock);
10248         for (i = 1; i < vnic->uc_filter_count; i++) {
10249                 struct hwrm_cfa_l2_filter_free_input req = {0};
10250
10251                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10252                                        -1);
10253
10254                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10255
10256                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10257                                         HWRM_CMD_TIMEOUT);
10258         }
10259         mutex_unlock(&bp->hwrm_cmd_lock);
10260
10261         vnic->uc_filter_count = 1;
10262
10263         netif_addr_lock_bh(dev);
10264         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10265                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10266         } else {
10267                 netdev_for_each_uc_addr(ha, dev) {
10268                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10269                         off += ETH_ALEN;
10270                         vnic->uc_filter_count++;
10271                 }
10272         }
10273         netif_addr_unlock_bh(dev);
10274
10275         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10276                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10277                 if (rc) {
10278                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10279                                    rc);
10280                         vnic->uc_filter_count = i;
10281                         return rc;
10282                 }
10283         }
10284
10285 skip_uc:
10286         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10287         if (rc && vnic->mc_list_count) {
10288                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10289                             rc);
10290                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10291                 vnic->mc_list_count = 0;
10292                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10293         }
10294         if (rc)
10295                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10296                            rc);
10297
10298         return rc;
10299 }
10300
10301 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10302 {
10303 #ifdef CONFIG_BNXT_SRIOV
10304         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10305                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10306
10307                 /* No minimum rings were provisioned by the PF.  Don't
10308                  * reserve rings by default when device is down.
10309                  */
10310                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10311                         return true;
10312
10313                 if (!netif_running(bp->dev))
10314                         return false;
10315         }
10316 #endif
10317         return true;
10318 }
10319
10320 /* If the chip and firmware supports RFS */
10321 static bool bnxt_rfs_supported(struct bnxt *bp)
10322 {
10323         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10324                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10325                         return true;
10326                 return false;
10327         }
10328         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10329                 return true;
10330         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10331                 return true;
10332         return false;
10333 }
10334
10335 /* If runtime conditions support RFS */
10336 static bool bnxt_rfs_capable(struct bnxt *bp)
10337 {
10338 #ifdef CONFIG_RFS_ACCEL
10339         int vnics, max_vnics, max_rss_ctxs;
10340
10341         if (bp->flags & BNXT_FLAG_CHIP_P5)
10342                 return bnxt_rfs_supported(bp);
10343         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10344                 return false;
10345
10346         vnics = 1 + bp->rx_nr_rings;
10347         max_vnics = bnxt_get_max_func_vnics(bp);
10348         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10349
10350         /* RSS contexts not a limiting factor */
10351         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10352                 max_rss_ctxs = max_vnics;
10353         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10354                 if (bp->rx_nr_rings > 1)
10355                         netdev_warn(bp->dev,
10356                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10357                                     min(max_rss_ctxs - 1, max_vnics - 1));
10358                 return false;
10359         }
10360
10361         if (!BNXT_NEW_RM(bp))
10362                 return true;
10363
10364         if (vnics == bp->hw_resc.resv_vnics)
10365                 return true;
10366
10367         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10368         if (vnics <= bp->hw_resc.resv_vnics)
10369                 return true;
10370
10371         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10372         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10373         return false;
10374 #else
10375         return false;
10376 #endif
10377 }
10378
10379 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10380                                            netdev_features_t features)
10381 {
10382         struct bnxt *bp = netdev_priv(dev);
10383         netdev_features_t vlan_features;
10384
10385         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10386                 features &= ~NETIF_F_NTUPLE;
10387
10388         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10389                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10390
10391         if (!(features & NETIF_F_GRO))
10392                 features &= ~NETIF_F_GRO_HW;
10393
10394         if (features & NETIF_F_GRO_HW)
10395                 features &= ~NETIF_F_LRO;
10396
10397         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10398          * turned on or off together.
10399          */
10400         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10401         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10402                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10403                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10404                 else if (vlan_features)
10405                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10406         }
10407 #ifdef CONFIG_BNXT_SRIOV
10408         if (BNXT_VF(bp) && bp->vf.vlan)
10409                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10410 #endif
10411         return features;
10412 }
10413
10414 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10415 {
10416         struct bnxt *bp = netdev_priv(dev);
10417         u32 flags = bp->flags;
10418         u32 changes;
10419         int rc = 0;
10420         bool re_init = false;
10421         bool update_tpa = false;
10422
10423         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10424         if (features & NETIF_F_GRO_HW)
10425                 flags |= BNXT_FLAG_GRO;
10426         else if (features & NETIF_F_LRO)
10427                 flags |= BNXT_FLAG_LRO;
10428
10429         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10430                 flags &= ~BNXT_FLAG_TPA;
10431
10432         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10433                 flags |= BNXT_FLAG_STRIP_VLAN;
10434
10435         if (features & NETIF_F_NTUPLE)
10436                 flags |= BNXT_FLAG_RFS;
10437
10438         changes = flags ^ bp->flags;
10439         if (changes & BNXT_FLAG_TPA) {
10440                 update_tpa = true;
10441                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10442                     (flags & BNXT_FLAG_TPA) == 0 ||
10443                     (bp->flags & BNXT_FLAG_CHIP_P5))
10444                         re_init = true;
10445         }
10446
10447         if (changes & ~BNXT_FLAG_TPA)
10448                 re_init = true;
10449
10450         if (flags != bp->flags) {
10451                 u32 old_flags = bp->flags;
10452
10453                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10454                         bp->flags = flags;
10455                         if (update_tpa)
10456                                 bnxt_set_ring_params(bp);
10457                         return rc;
10458                 }
10459
10460                 if (re_init) {
10461                         bnxt_close_nic(bp, false, false);
10462                         bp->flags = flags;
10463                         if (update_tpa)
10464                                 bnxt_set_ring_params(bp);
10465
10466                         return bnxt_open_nic(bp, false, false);
10467                 }
10468                 if (update_tpa) {
10469                         bp->flags = flags;
10470                         rc = bnxt_set_tpa(bp,
10471                                           (flags & BNXT_FLAG_TPA) ?
10472                                           true : false);
10473                         if (rc)
10474                                 bp->flags = old_flags;
10475                 }
10476         }
10477         return rc;
10478 }
10479
10480 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10481                          u32 *reg_buf)
10482 {
10483         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10484         struct hwrm_dbg_read_direct_input req = {0};
10485         __le32 *dbg_reg_buf;
10486         dma_addr_t mapping;
10487         int rc, i;
10488
10489         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10490                                          &mapping, GFP_KERNEL);
10491         if (!dbg_reg_buf)
10492                 return -ENOMEM;
10493         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10494         req.host_dest_addr = cpu_to_le64(mapping);
10495         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10496         req.read_len32 = cpu_to_le32(num_words);
10497         mutex_lock(&bp->hwrm_cmd_lock);
10498         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10499         if (rc || resp->error_code) {
10500                 rc = -EIO;
10501                 goto dbg_rd_reg_exit;
10502         }
10503         for (i = 0; i < num_words; i++)
10504                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10505
10506 dbg_rd_reg_exit:
10507         mutex_unlock(&bp->hwrm_cmd_lock);
10508         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10509         return rc;
10510 }
10511
10512 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10513                                        u32 ring_id, u32 *prod, u32 *cons)
10514 {
10515         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10516         struct hwrm_dbg_ring_info_get_input req = {0};
10517         int rc;
10518
10519         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10520         req.ring_type = ring_type;
10521         req.fw_ring_id = cpu_to_le32(ring_id);
10522         mutex_lock(&bp->hwrm_cmd_lock);
10523         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10524         if (!rc) {
10525                 *prod = le32_to_cpu(resp->producer_index);
10526                 *cons = le32_to_cpu(resp->consumer_index);
10527         }
10528         mutex_unlock(&bp->hwrm_cmd_lock);
10529         return rc;
10530 }
10531
10532 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10533 {
10534         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10535         int i = bnapi->index;
10536
10537         if (!txr)
10538                 return;
10539
10540         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10541                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10542                     txr->tx_cons);
10543 }
10544
10545 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10546 {
10547         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10548         int i = bnapi->index;
10549
10550         if (!rxr)
10551                 return;
10552
10553         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10554                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10555                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10556                     rxr->rx_sw_agg_prod);
10557 }
10558
10559 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10560 {
10561         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10562         int i = bnapi->index;
10563
10564         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10565                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10566 }
10567
10568 static void bnxt_dbg_dump_states(struct bnxt *bp)
10569 {
10570         int i;
10571         struct bnxt_napi *bnapi;
10572
10573         for (i = 0; i < bp->cp_nr_rings; i++) {
10574                 bnapi = bp->bnapi[i];
10575                 if (netif_msg_drv(bp)) {
10576                         bnxt_dump_tx_sw_state(bnapi);
10577                         bnxt_dump_rx_sw_state(bnapi);
10578                         bnxt_dump_cp_sw_state(bnapi);
10579                 }
10580         }
10581 }
10582
10583 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
10584 {
10585         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
10586         struct hwrm_ring_reset_input req = {0};
10587         struct bnxt_napi *bnapi = rxr->bnapi;
10588         struct bnxt_cp_ring_info *cpr;
10589         u16 cp_ring_id;
10590
10591         cpr = &bnapi->cp_ring;
10592         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
10593         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
10594         req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
10595         req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
10596         return hwrm_send_message_silent(bp, &req, sizeof(req),
10597                                         HWRM_CMD_TIMEOUT);
10598 }
10599
10600 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10601 {
10602         if (!silent)
10603                 bnxt_dbg_dump_states(bp);
10604         if (netif_running(bp->dev)) {
10605                 int rc;
10606
10607                 if (silent) {
10608                         bnxt_close_nic(bp, false, false);
10609                         bnxt_open_nic(bp, false, false);
10610                 } else {
10611                         bnxt_ulp_stop(bp);
10612                         bnxt_close_nic(bp, true, false);
10613                         rc = bnxt_open_nic(bp, true, false);
10614                         bnxt_ulp_start(bp, rc);
10615                 }
10616         }
10617 }
10618
10619 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10620 {
10621         struct bnxt *bp = netdev_priv(dev);
10622
10623         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10624         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10625         bnxt_queue_sp_work(bp);
10626 }
10627
10628 static void bnxt_fw_health_check(struct bnxt *bp)
10629 {
10630         struct bnxt_fw_health *fw_health = bp->fw_health;
10631         u32 val;
10632
10633         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10634                 return;
10635
10636         if (fw_health->tmr_counter) {
10637                 fw_health->tmr_counter--;
10638                 return;
10639         }
10640
10641         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10642         if (val == fw_health->last_fw_heartbeat)
10643                 goto fw_reset;
10644
10645         fw_health->last_fw_heartbeat = val;
10646
10647         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10648         if (val != fw_health->last_fw_reset_cnt)
10649                 goto fw_reset;
10650
10651         fw_health->tmr_counter = fw_health->tmr_multiplier;
10652         return;
10653
10654 fw_reset:
10655         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10656         bnxt_queue_sp_work(bp);
10657 }
10658
10659 static void bnxt_timer(struct timer_list *t)
10660 {
10661         struct bnxt *bp = from_timer(bp, t, timer);
10662         struct net_device *dev = bp->dev;
10663
10664         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10665                 return;
10666
10667         if (atomic_read(&bp->intr_sem) != 0)
10668                 goto bnxt_restart_timer;
10669
10670         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10671                 bnxt_fw_health_check(bp);
10672
10673         if (bp->link_info.link_up && bp->stats_coal_ticks) {
10674                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10675                 bnxt_queue_sp_work(bp);
10676         }
10677
10678         if (bnxt_tc_flower_enabled(bp)) {
10679                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10680                 bnxt_queue_sp_work(bp);
10681         }
10682
10683 #ifdef CONFIG_RFS_ACCEL
10684         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10685                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10686                 bnxt_queue_sp_work(bp);
10687         }
10688 #endif /*CONFIG_RFS_ACCEL*/
10689
10690         if (bp->link_info.phy_retry) {
10691                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10692                         bp->link_info.phy_retry = false;
10693                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10694                 } else {
10695                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10696                         bnxt_queue_sp_work(bp);
10697                 }
10698         }
10699
10700         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10701             netif_carrier_ok(dev)) {
10702                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10703                 bnxt_queue_sp_work(bp);
10704         }
10705 bnxt_restart_timer:
10706         mod_timer(&bp->timer, jiffies + bp->current_interval);
10707 }
10708
10709 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10710 {
10711         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10712          * set.  If the device is being closed, bnxt_close() may be holding
10713          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10714          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10715          */
10716         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10717         rtnl_lock();
10718 }
10719
10720 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10721 {
10722         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10723         rtnl_unlock();
10724 }
10725
10726 /* Only called from bnxt_sp_task() */
10727 static void bnxt_reset(struct bnxt *bp, bool silent)
10728 {
10729         bnxt_rtnl_lock_sp(bp);
10730         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10731                 bnxt_reset_task(bp, silent);
10732         bnxt_rtnl_unlock_sp(bp);
10733 }
10734
10735 /* Only called from bnxt_sp_task() */
10736 static void bnxt_rx_ring_reset(struct bnxt *bp)
10737 {
10738         int i;
10739
10740         bnxt_rtnl_lock_sp(bp);
10741         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10742                 bnxt_rtnl_unlock_sp(bp);
10743                 return;
10744         }
10745         /* Disable and flush TPA before resetting the RX ring */
10746         if (bp->flags & BNXT_FLAG_TPA)
10747                 bnxt_set_tpa(bp, false);
10748         for (i = 0; i < bp->rx_nr_rings; i++) {
10749                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
10750                 struct bnxt_cp_ring_info *cpr;
10751                 int rc;
10752
10753                 if (!rxr->bnapi->in_reset)
10754                         continue;
10755
10756                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
10757                 if (rc) {
10758                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
10759                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
10760                         else
10761                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
10762                                             rc);
10763                         bnxt_reset_task(bp, true);
10764                         break;
10765                 }
10766                 bnxt_free_one_rx_ring_skbs(bp, i);
10767                 rxr->rx_prod = 0;
10768                 rxr->rx_agg_prod = 0;
10769                 rxr->rx_sw_agg_prod = 0;
10770                 rxr->rx_next_cons = 0;
10771                 rxr->bnapi->in_reset = false;
10772                 bnxt_alloc_one_rx_ring(bp, i);
10773                 cpr = &rxr->bnapi->cp_ring;
10774                 cpr->sw_stats.rx.rx_resets++;
10775                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10776                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10777                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10778         }
10779         if (bp->flags & BNXT_FLAG_TPA)
10780                 bnxt_set_tpa(bp, true);
10781         bnxt_rtnl_unlock_sp(bp);
10782 }
10783
10784 static void bnxt_fw_reset_close(struct bnxt *bp)
10785 {
10786         bnxt_ulp_stop(bp);
10787         /* When firmware is fatal state, disable PCI device to prevent
10788          * any potential bad DMAs before freeing kernel memory.
10789          */
10790         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10791                 pci_disable_device(bp->pdev);
10792         __bnxt_close_nic(bp, true, false);
10793         bnxt_clear_int_mode(bp);
10794         bnxt_hwrm_func_drv_unrgtr(bp);
10795         if (pci_is_enabled(bp->pdev))
10796                 pci_disable_device(bp->pdev);
10797         bnxt_free_ctx_mem(bp);
10798         kfree(bp->ctx);
10799         bp->ctx = NULL;
10800 }
10801
10802 static bool is_bnxt_fw_ok(struct bnxt *bp)
10803 {
10804         struct bnxt_fw_health *fw_health = bp->fw_health;
10805         bool no_heartbeat = false, has_reset = false;
10806         u32 val;
10807
10808         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10809         if (val == fw_health->last_fw_heartbeat)
10810                 no_heartbeat = true;
10811
10812         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10813         if (val != fw_health->last_fw_reset_cnt)
10814                 has_reset = true;
10815
10816         if (!no_heartbeat && has_reset)
10817                 return true;
10818
10819         return false;
10820 }
10821
10822 /* rtnl_lock is acquired before calling this function */
10823 static void bnxt_force_fw_reset(struct bnxt *bp)
10824 {
10825         struct bnxt_fw_health *fw_health = bp->fw_health;
10826         u32 wait_dsecs;
10827
10828         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10829             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10830                 return;
10831
10832         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10833         bnxt_fw_reset_close(bp);
10834         wait_dsecs = fw_health->master_func_wait_dsecs;
10835         if (fw_health->master) {
10836                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10837                         wait_dsecs = 0;
10838                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10839         } else {
10840                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10841                 wait_dsecs = fw_health->normal_func_wait_dsecs;
10842                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10843         }
10844
10845         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10846         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10847         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10848 }
10849
10850 void bnxt_fw_exception(struct bnxt *bp)
10851 {
10852         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10853         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10854         bnxt_rtnl_lock_sp(bp);
10855         bnxt_force_fw_reset(bp);
10856         bnxt_rtnl_unlock_sp(bp);
10857 }
10858
10859 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10860  * < 0 on error.
10861  */
10862 static int bnxt_get_registered_vfs(struct bnxt *bp)
10863 {
10864 #ifdef CONFIG_BNXT_SRIOV
10865         int rc;
10866
10867         if (!BNXT_PF(bp))
10868                 return 0;
10869
10870         rc = bnxt_hwrm_func_qcfg(bp);
10871         if (rc) {
10872                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10873                 return rc;
10874         }
10875         if (bp->pf.registered_vfs)
10876                 return bp->pf.registered_vfs;
10877         if (bp->sriov_cfg)
10878                 return 1;
10879 #endif
10880         return 0;
10881 }
10882
10883 void bnxt_fw_reset(struct bnxt *bp)
10884 {
10885         bnxt_rtnl_lock_sp(bp);
10886         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10887             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10888                 int n = 0, tmo;
10889
10890                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10891                 if (bp->pf.active_vfs &&
10892                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10893                         n = bnxt_get_registered_vfs(bp);
10894                 if (n < 0) {
10895                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10896                                    n);
10897                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10898                         dev_close(bp->dev);
10899                         goto fw_reset_exit;
10900                 } else if (n > 0) {
10901                         u16 vf_tmo_dsecs = n * 10;
10902
10903                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10904                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10905                         bp->fw_reset_state =
10906                                 BNXT_FW_RESET_STATE_POLL_VF;
10907                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10908                         goto fw_reset_exit;
10909                 }
10910                 bnxt_fw_reset_close(bp);
10911                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10912                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10913                         tmo = HZ / 10;
10914                 } else {
10915                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10916                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10917                 }
10918                 bnxt_queue_fw_reset_work(bp, tmo);
10919         }
10920 fw_reset_exit:
10921         bnxt_rtnl_unlock_sp(bp);
10922 }
10923
10924 static void bnxt_chk_missed_irq(struct bnxt *bp)
10925 {
10926         int i;
10927
10928         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10929                 return;
10930
10931         for (i = 0; i < bp->cp_nr_rings; i++) {
10932                 struct bnxt_napi *bnapi = bp->bnapi[i];
10933                 struct bnxt_cp_ring_info *cpr;
10934                 u32 fw_ring_id;
10935                 int j;
10936
10937                 if (!bnapi)
10938                         continue;
10939
10940                 cpr = &bnapi->cp_ring;
10941                 for (j = 0; j < 2; j++) {
10942                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10943                         u32 val[2];
10944
10945                         if (!cpr2 || cpr2->has_more_work ||
10946                             !bnxt_has_work(bp, cpr2))
10947                                 continue;
10948
10949                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10950                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10951                                 continue;
10952                         }
10953                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10954                         bnxt_dbg_hwrm_ring_info_get(bp,
10955                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10956                                 fw_ring_id, &val[0], &val[1]);
10957                         cpr->sw_stats.cmn.missed_irqs++;
10958                 }
10959         }
10960 }
10961
10962 static void bnxt_cfg_ntp_filters(struct bnxt *);
10963
10964 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10965 {
10966         struct bnxt_link_info *link_info = &bp->link_info;
10967
10968         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10969                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10970                 if (bp->hwrm_spec_code >= 0x10201) {
10971                         if (link_info->auto_pause_setting &
10972                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10973                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10974                 } else {
10975                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10976                 }
10977                 link_info->advertising = link_info->auto_link_speeds;
10978                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
10979         } else {
10980                 link_info->req_link_speed = link_info->force_link_speed;
10981                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
10982                 if (link_info->force_pam4_link_speed) {
10983                         link_info->req_link_speed =
10984                                 link_info->force_pam4_link_speed;
10985                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
10986                 }
10987                 link_info->req_duplex = link_info->duplex_setting;
10988         }
10989         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10990                 link_info->req_flow_ctrl =
10991                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10992         else
10993                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10994 }
10995
10996 static void bnxt_sp_task(struct work_struct *work)
10997 {
10998         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10999
11000         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11001         smp_mb__after_atomic();
11002         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11003                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11004                 return;
11005         }
11006
11007         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11008                 bnxt_cfg_rx_mode(bp);
11009
11010         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11011                 bnxt_cfg_ntp_filters(bp);
11012         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11013                 bnxt_hwrm_exec_fwd_req(bp);
11014         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11015                 bnxt_hwrm_port_qstats(bp, 0);
11016                 bnxt_hwrm_port_qstats_ext(bp, 0);
11017                 bnxt_accumulate_all_stats(bp);
11018         }
11019
11020         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11021                 int rc;
11022
11023                 mutex_lock(&bp->link_lock);
11024                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11025                                        &bp->sp_event))
11026                         bnxt_hwrm_phy_qcaps(bp);
11027
11028                 rc = bnxt_update_link(bp, true);
11029                 if (rc)
11030                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11031                                    rc);
11032
11033                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11034                                        &bp->sp_event))
11035                         bnxt_init_ethtool_link_settings(bp);
11036                 mutex_unlock(&bp->link_lock);
11037         }
11038         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11039                 int rc;
11040
11041                 mutex_lock(&bp->link_lock);
11042                 rc = bnxt_update_phy_setting(bp);
11043                 mutex_unlock(&bp->link_lock);
11044                 if (rc) {
11045                         netdev_warn(bp->dev, "update phy settings retry failed\n");
11046                 } else {
11047                         bp->link_info.phy_retry = false;
11048                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
11049                 }
11050         }
11051         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11052                 mutex_lock(&bp->link_lock);
11053                 bnxt_get_port_module_status(bp);
11054                 mutex_unlock(&bp->link_lock);
11055         }
11056
11057         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11058                 bnxt_tc_flow_stats_work(bp);
11059
11060         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11061                 bnxt_chk_missed_irq(bp);
11062
11063         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11064          * must be the last functions to be called before exiting.
11065          */
11066         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11067                 bnxt_reset(bp, false);
11068
11069         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11070                 bnxt_reset(bp, true);
11071
11072         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11073                 bnxt_rx_ring_reset(bp);
11074
11075         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11076                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11077
11078         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11079                 if (!is_bnxt_fw_ok(bp))
11080                         bnxt_devlink_health_report(bp,
11081                                                    BNXT_FW_EXCEPTION_SP_EVENT);
11082         }
11083
11084         smp_mb__before_atomic();
11085         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11086 }
11087
11088 /* Under rtnl_lock */
11089 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11090                      int tx_xdp)
11091 {
11092         int max_rx, max_tx, tx_sets = 1;
11093         int tx_rings_needed, stats;
11094         int rx_rings = rx;
11095         int cp, vnics, rc;
11096
11097         if (tcs)
11098                 tx_sets = tcs;
11099
11100         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11101         if (rc)
11102                 return rc;
11103
11104         if (max_rx < rx)
11105                 return -ENOMEM;
11106
11107         tx_rings_needed = tx * tx_sets + tx_xdp;
11108         if (max_tx < tx_rings_needed)
11109                 return -ENOMEM;
11110
11111         vnics = 1;
11112         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11113                 vnics += rx_rings;
11114
11115         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11116                 rx_rings <<= 1;
11117         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11118         stats = cp;
11119         if (BNXT_NEW_RM(bp)) {
11120                 cp += bnxt_get_ulp_msix_num(bp);
11121                 stats += bnxt_get_ulp_stat_ctxs(bp);
11122         }
11123         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11124                                      stats, vnics);
11125 }
11126
11127 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11128 {
11129         if (bp->bar2) {
11130                 pci_iounmap(pdev, bp->bar2);
11131                 bp->bar2 = NULL;
11132         }
11133
11134         if (bp->bar1) {
11135                 pci_iounmap(pdev, bp->bar1);
11136                 bp->bar1 = NULL;
11137         }
11138
11139         if (bp->bar0) {
11140                 pci_iounmap(pdev, bp->bar0);
11141                 bp->bar0 = NULL;
11142         }
11143 }
11144
11145 static void bnxt_cleanup_pci(struct bnxt *bp)
11146 {
11147         bnxt_unmap_bars(bp, bp->pdev);
11148         pci_release_regions(bp->pdev);
11149         if (pci_is_enabled(bp->pdev))
11150                 pci_disable_device(bp->pdev);
11151 }
11152
11153 static void bnxt_init_dflt_coal(struct bnxt *bp)
11154 {
11155         struct bnxt_coal *coal;
11156
11157         /* Tick values in micro seconds.
11158          * 1 coal_buf x bufs_per_record = 1 completion record.
11159          */
11160         coal = &bp->rx_coal;
11161         coal->coal_ticks = 10;
11162         coal->coal_bufs = 30;
11163         coal->coal_ticks_irq = 1;
11164         coal->coal_bufs_irq = 2;
11165         coal->idle_thresh = 50;
11166         coal->bufs_per_record = 2;
11167         coal->budget = 64;              /* NAPI budget */
11168
11169         coal = &bp->tx_coal;
11170         coal->coal_ticks = 28;
11171         coal->coal_bufs = 30;
11172         coal->coal_ticks_irq = 2;
11173         coal->coal_bufs_irq = 2;
11174         coal->bufs_per_record = 1;
11175
11176         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11177 }
11178
11179 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11180 {
11181 #ifdef CONFIG_TEE_BNXT_FW
11182         int rc = tee_bnxt_fw_load();
11183
11184         if (rc)
11185                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11186
11187         return rc;
11188 #else
11189         netdev_err(bp->dev, "OP-TEE not supported\n");
11190         return -ENODEV;
11191 #endif
11192 }
11193
11194 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11195 {
11196         int rc;
11197
11198         bp->fw_cap = 0;
11199         rc = bnxt_hwrm_ver_get(bp);
11200         bnxt_try_map_fw_health_reg(bp);
11201         if (rc) {
11202                 if (bp->fw_health && bp->fw_health->status_reliable) {
11203                         u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11204
11205                         netdev_err(bp->dev,
11206                                    "Firmware not responding, status: 0x%x\n",
11207                                    sts);
11208                         if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11209                                 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11210                                 rc = bnxt_fw_reset_via_optee(bp);
11211                                 if (!rc)
11212                                         rc = bnxt_hwrm_ver_get(bp);
11213                         }
11214                 }
11215                 if (rc)
11216                         return rc;
11217         }
11218
11219         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11220                 rc = bnxt_alloc_kong_hwrm_resources(bp);
11221                 if (rc)
11222                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11223         }
11224
11225         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11226             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11227                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11228                 if (rc)
11229                         return rc;
11230         }
11231         bnxt_nvm_cfg_ver_get(bp);
11232
11233         rc = bnxt_hwrm_func_reset(bp);
11234         if (rc)
11235                 return -ENODEV;
11236
11237         bnxt_hwrm_fw_set_time(bp);
11238         return 0;
11239 }
11240
11241 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11242 {
11243         int rc;
11244
11245         /* Get the MAX capabilities for this function */
11246         rc = bnxt_hwrm_func_qcaps(bp);
11247         if (rc) {
11248                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11249                            rc);
11250                 return -ENODEV;
11251         }
11252
11253         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11254         if (rc)
11255                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11256                             rc);
11257
11258         if (bnxt_alloc_fw_health(bp)) {
11259                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11260         } else {
11261                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11262                 if (rc)
11263                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11264                                     rc);
11265         }
11266
11267         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11268         if (rc)
11269                 return -ENODEV;
11270
11271         bnxt_hwrm_func_qcfg(bp);
11272         bnxt_hwrm_vnic_qcaps(bp);
11273         bnxt_hwrm_port_led_qcaps(bp);
11274         bnxt_ethtool_init(bp);
11275         bnxt_dcb_init(bp);
11276         return 0;
11277 }
11278
11279 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11280 {
11281         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11282         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11283                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11284                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11285                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11286         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11287                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11288                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11289                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11290         }
11291 }
11292
11293 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11294 {
11295         struct net_device *dev = bp->dev;
11296
11297         dev->hw_features &= ~NETIF_F_NTUPLE;
11298         dev->features &= ~NETIF_F_NTUPLE;
11299         bp->flags &= ~BNXT_FLAG_RFS;
11300         if (bnxt_rfs_supported(bp)) {
11301                 dev->hw_features |= NETIF_F_NTUPLE;
11302                 if (bnxt_rfs_capable(bp)) {
11303                         bp->flags |= BNXT_FLAG_RFS;
11304                         dev->features |= NETIF_F_NTUPLE;
11305                 }
11306         }
11307 }
11308
11309 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11310 {
11311         struct pci_dev *pdev = bp->pdev;
11312
11313         bnxt_set_dflt_rss_hash_type(bp);
11314         bnxt_set_dflt_rfs(bp);
11315
11316         bnxt_get_wol_settings(bp);
11317         if (bp->flags & BNXT_FLAG_WOL_CAP)
11318                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11319         else
11320                 device_set_wakeup_capable(&pdev->dev, false);
11321
11322         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11323         bnxt_hwrm_coal_params_qcaps(bp);
11324 }
11325
11326 static int bnxt_fw_init_one(struct bnxt *bp)
11327 {
11328         int rc;
11329
11330         rc = bnxt_fw_init_one_p1(bp);
11331         if (rc) {
11332                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11333                 return rc;
11334         }
11335         rc = bnxt_fw_init_one_p2(bp);
11336         if (rc) {
11337                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11338                 return rc;
11339         }
11340         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11341         if (rc)
11342                 return rc;
11343
11344         /* In case fw capabilities have changed, destroy the unneeded
11345          * reporters and create newly capable ones.
11346          */
11347         bnxt_dl_fw_reporters_destroy(bp, false);
11348         bnxt_dl_fw_reporters_create(bp);
11349         bnxt_fw_init_one_p3(bp);
11350         return 0;
11351 }
11352
11353 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11354 {
11355         struct bnxt_fw_health *fw_health = bp->fw_health;
11356         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11357         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11358         u32 reg_type, reg_off, delay_msecs;
11359
11360         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11361         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11362         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11363         switch (reg_type) {
11364         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11365                 pci_write_config_dword(bp->pdev, reg_off, val);
11366                 break;
11367         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11368                 writel(reg_off & BNXT_GRC_BASE_MASK,
11369                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11370                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11371                 fallthrough;
11372         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11373                 writel(val, bp->bar0 + reg_off);
11374                 break;
11375         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11376                 writel(val, bp->bar1 + reg_off);
11377                 break;
11378         }
11379         if (delay_msecs) {
11380                 pci_read_config_dword(bp->pdev, 0, &val);
11381                 msleep(delay_msecs);
11382         }
11383 }
11384
11385 static void bnxt_reset_all(struct bnxt *bp)
11386 {
11387         struct bnxt_fw_health *fw_health = bp->fw_health;
11388         int i, rc;
11389
11390         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11391                 bnxt_fw_reset_via_optee(bp);
11392                 bp->fw_reset_timestamp = jiffies;
11393                 return;
11394         }
11395
11396         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11397                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11398                         bnxt_fw_reset_writel(bp, i);
11399         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11400                 struct hwrm_fw_reset_input req = {0};
11401
11402                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11403                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11404                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11405                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11406                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11407                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11408                 if (rc)
11409                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11410         }
11411         bp->fw_reset_timestamp = jiffies;
11412 }
11413
11414 static void bnxt_fw_reset_task(struct work_struct *work)
11415 {
11416         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11417         int rc;
11418
11419         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11420                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11421                 return;
11422         }
11423
11424         switch (bp->fw_reset_state) {
11425         case BNXT_FW_RESET_STATE_POLL_VF: {
11426                 int n = bnxt_get_registered_vfs(bp);
11427                 int tmo;
11428
11429                 if (n < 0) {
11430                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11431                                    n, jiffies_to_msecs(jiffies -
11432                                    bp->fw_reset_timestamp));
11433                         goto fw_reset_abort;
11434                 } else if (n > 0) {
11435                         if (time_after(jiffies, bp->fw_reset_timestamp +
11436                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11437                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11438                                 bp->fw_reset_state = 0;
11439                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11440                                            n);
11441                                 return;
11442                         }
11443                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11444                         return;
11445                 }
11446                 bp->fw_reset_timestamp = jiffies;
11447                 rtnl_lock();
11448                 bnxt_fw_reset_close(bp);
11449                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11450                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11451                         tmo = HZ / 10;
11452                 } else {
11453                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11454                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11455                 }
11456                 rtnl_unlock();
11457                 bnxt_queue_fw_reset_work(bp, tmo);
11458                 return;
11459         }
11460         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11461                 u32 val;
11462
11463                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11464                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11465                     !time_after(jiffies, bp->fw_reset_timestamp +
11466                     (bp->fw_reset_max_dsecs * HZ / 10))) {
11467                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11468                         return;
11469                 }
11470
11471                 if (!bp->fw_health->master) {
11472                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11473
11474                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11475                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11476                         return;
11477                 }
11478                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11479         }
11480                 fallthrough;
11481         case BNXT_FW_RESET_STATE_RESET_FW:
11482                 bnxt_reset_all(bp);
11483                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11484                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11485                 return;
11486         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11487                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11488                         u32 val;
11489
11490                         val = bnxt_fw_health_readl(bp,
11491                                                    BNXT_FW_RESET_INPROG_REG);
11492                         if (val)
11493                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11494                                             val);
11495                 }
11496                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11497                 if (pci_enable_device(bp->pdev)) {
11498                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11499                         goto fw_reset_abort;
11500                 }
11501                 pci_set_master(bp->pdev);
11502                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11503                 fallthrough;
11504         case BNXT_FW_RESET_STATE_POLL_FW:
11505                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11506                 rc = __bnxt_hwrm_ver_get(bp, true);
11507                 if (rc) {
11508                         if (time_after(jiffies, bp->fw_reset_timestamp +
11509                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11510                                 netdev_err(bp->dev, "Firmware reset aborted\n");
11511                                 goto fw_reset_abort_status;
11512                         }
11513                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11514                         return;
11515                 }
11516                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11517                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11518                 fallthrough;
11519         case BNXT_FW_RESET_STATE_OPENING:
11520                 while (!rtnl_trylock()) {
11521                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11522                         return;
11523                 }
11524                 rc = bnxt_open(bp->dev);
11525                 if (rc) {
11526                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11527                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11528                         dev_close(bp->dev);
11529                 }
11530
11531                 bp->fw_reset_state = 0;
11532                 /* Make sure fw_reset_state is 0 before clearing the flag */
11533                 smp_mb__before_atomic();
11534                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11535                 bnxt_ulp_start(bp, rc);
11536                 if (!rc)
11537                         bnxt_reenable_sriov(bp);
11538                 bnxt_dl_health_recovery_done(bp);
11539                 bnxt_dl_health_status_update(bp, true);
11540                 rtnl_unlock();
11541                 break;
11542         }
11543         return;
11544
11545 fw_reset_abort_status:
11546         if (bp->fw_health->status_reliable ||
11547             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
11548                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11549
11550                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
11551         }
11552 fw_reset_abort:
11553         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11554         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11555                 bnxt_dl_health_status_update(bp, false);
11556         bp->fw_reset_state = 0;
11557         rtnl_lock();
11558         dev_close(bp->dev);
11559         rtnl_unlock();
11560 }
11561
11562 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11563 {
11564         int rc;
11565         struct bnxt *bp = netdev_priv(dev);
11566
11567         SET_NETDEV_DEV(dev, &pdev->dev);
11568
11569         /* enable device (incl. PCI PM wakeup), and bus-mastering */
11570         rc = pci_enable_device(pdev);
11571         if (rc) {
11572                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11573                 goto init_err;
11574         }
11575
11576         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11577                 dev_err(&pdev->dev,
11578                         "Cannot find PCI device base address, aborting\n");
11579                 rc = -ENODEV;
11580                 goto init_err_disable;
11581         }
11582
11583         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11584         if (rc) {
11585                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11586                 goto init_err_disable;
11587         }
11588
11589         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11590             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11591                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11592                 goto init_err_disable;
11593         }
11594
11595         pci_set_master(pdev);
11596
11597         bp->dev = dev;
11598         bp->pdev = pdev;
11599
11600         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11601          * determines the BAR size.
11602          */
11603         bp->bar0 = pci_ioremap_bar(pdev, 0);
11604         if (!bp->bar0) {
11605                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11606                 rc = -ENOMEM;
11607                 goto init_err_release;
11608         }
11609
11610         bp->bar2 = pci_ioremap_bar(pdev, 4);
11611         if (!bp->bar2) {
11612                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11613                 rc = -ENOMEM;
11614                 goto init_err_release;
11615         }
11616
11617         pci_enable_pcie_error_reporting(pdev);
11618
11619         INIT_WORK(&bp->sp_task, bnxt_sp_task);
11620         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11621
11622         spin_lock_init(&bp->ntp_fltr_lock);
11623 #if BITS_PER_LONG == 32
11624         spin_lock_init(&bp->db_lock);
11625 #endif
11626
11627         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11628         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11629
11630         bnxt_init_dflt_coal(bp);
11631
11632         timer_setup(&bp->timer, bnxt_timer, 0);
11633         bp->current_interval = BNXT_TIMER_INTERVAL;
11634
11635         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11636         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11637
11638         clear_bit(BNXT_STATE_OPEN, &bp->state);
11639         return 0;
11640
11641 init_err_release:
11642         bnxt_unmap_bars(bp, pdev);
11643         pci_release_regions(pdev);
11644
11645 init_err_disable:
11646         pci_disable_device(pdev);
11647
11648 init_err:
11649         return rc;
11650 }
11651
11652 /* rtnl_lock held */
11653 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11654 {
11655         struct sockaddr *addr = p;
11656         struct bnxt *bp = netdev_priv(dev);
11657         int rc = 0;
11658
11659         if (!is_valid_ether_addr(addr->sa_data))
11660                 return -EADDRNOTAVAIL;
11661
11662         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11663                 return 0;
11664
11665         rc = bnxt_approve_mac(bp, addr->sa_data, true);
11666         if (rc)
11667                 return rc;
11668
11669         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11670         if (netif_running(dev)) {
11671                 bnxt_close_nic(bp, false, false);
11672                 rc = bnxt_open_nic(bp, false, false);
11673         }
11674
11675         return rc;
11676 }
11677
11678 /* rtnl_lock held */
11679 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11680 {
11681         struct bnxt *bp = netdev_priv(dev);
11682
11683         if (netif_running(dev))
11684                 bnxt_close_nic(bp, true, false);
11685
11686         dev->mtu = new_mtu;
11687         bnxt_set_ring_params(bp);
11688
11689         if (netif_running(dev))
11690                 return bnxt_open_nic(bp, true, false);
11691
11692         return 0;
11693 }
11694
11695 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11696 {
11697         struct bnxt *bp = netdev_priv(dev);
11698         bool sh = false;
11699         int rc;
11700
11701         if (tc > bp->max_tc) {
11702                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11703                            tc, bp->max_tc);
11704                 return -EINVAL;
11705         }
11706
11707         if (netdev_get_num_tc(dev) == tc)
11708                 return 0;
11709
11710         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11711                 sh = true;
11712
11713         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11714                               sh, tc, bp->tx_nr_rings_xdp);
11715         if (rc)
11716                 return rc;
11717
11718         /* Needs to close the device and do hw resource re-allocations */
11719         if (netif_running(bp->dev))
11720                 bnxt_close_nic(bp, true, false);
11721
11722         if (tc) {
11723                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11724                 netdev_set_num_tc(dev, tc);
11725         } else {
11726                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11727                 netdev_reset_tc(dev);
11728         }
11729         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11730         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11731                                bp->tx_nr_rings + bp->rx_nr_rings;
11732
11733         if (netif_running(bp->dev))
11734                 return bnxt_open_nic(bp, true, false);
11735
11736         return 0;
11737 }
11738
11739 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11740                                   void *cb_priv)
11741 {
11742         struct bnxt *bp = cb_priv;
11743
11744         if (!bnxt_tc_flower_enabled(bp) ||
11745             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11746                 return -EOPNOTSUPP;
11747
11748         switch (type) {
11749         case TC_SETUP_CLSFLOWER:
11750                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11751         default:
11752                 return -EOPNOTSUPP;
11753         }
11754 }
11755
11756 LIST_HEAD(bnxt_block_cb_list);
11757
11758 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11759                          void *type_data)
11760 {
11761         struct bnxt *bp = netdev_priv(dev);
11762
11763         switch (type) {
11764         case TC_SETUP_BLOCK:
11765                 return flow_block_cb_setup_simple(type_data,
11766                                                   &bnxt_block_cb_list,
11767                                                   bnxt_setup_tc_block_cb,
11768                                                   bp, bp, true);
11769         case TC_SETUP_QDISC_MQPRIO: {
11770                 struct tc_mqprio_qopt *mqprio = type_data;
11771
11772                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11773
11774                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11775         }
11776         default:
11777                 return -EOPNOTSUPP;
11778         }
11779 }
11780
11781 #ifdef CONFIG_RFS_ACCEL
11782 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11783                             struct bnxt_ntuple_filter *f2)
11784 {
11785         struct flow_keys *keys1 = &f1->fkeys;
11786         struct flow_keys *keys2 = &f2->fkeys;
11787
11788         if (keys1->basic.n_proto != keys2->basic.n_proto ||
11789             keys1->basic.ip_proto != keys2->basic.ip_proto)
11790                 return false;
11791
11792         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11793                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11794                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11795                         return false;
11796         } else {
11797                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11798                            sizeof(keys1->addrs.v6addrs.src)) ||
11799                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11800                            sizeof(keys1->addrs.v6addrs.dst)))
11801                         return false;
11802         }
11803
11804         if (keys1->ports.ports == keys2->ports.ports &&
11805             keys1->control.flags == keys2->control.flags &&
11806             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11807             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11808                 return true;
11809
11810         return false;
11811 }
11812
11813 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11814                               u16 rxq_index, u32 flow_id)
11815 {
11816         struct bnxt *bp = netdev_priv(dev);
11817         struct bnxt_ntuple_filter *fltr, *new_fltr;
11818         struct flow_keys *fkeys;
11819         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11820         int rc = 0, idx, bit_id, l2_idx = 0;
11821         struct hlist_head *head;
11822         u32 flags;
11823
11824         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11825                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11826                 int off = 0, j;
11827
11828                 netif_addr_lock_bh(dev);
11829                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11830                         if (ether_addr_equal(eth->h_dest,
11831                                              vnic->uc_list + off)) {
11832                                 l2_idx = j + 1;
11833                                 break;
11834                         }
11835                 }
11836                 netif_addr_unlock_bh(dev);
11837                 if (!l2_idx)
11838                         return -EINVAL;
11839         }
11840         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11841         if (!new_fltr)
11842                 return -ENOMEM;
11843
11844         fkeys = &new_fltr->fkeys;
11845         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11846                 rc = -EPROTONOSUPPORT;
11847                 goto err_free;
11848         }
11849
11850         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11851              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11852             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11853              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11854                 rc = -EPROTONOSUPPORT;
11855                 goto err_free;
11856         }
11857         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11858             bp->hwrm_spec_code < 0x10601) {
11859                 rc = -EPROTONOSUPPORT;
11860                 goto err_free;
11861         }
11862         flags = fkeys->control.flags;
11863         if (((flags & FLOW_DIS_ENCAPSULATION) &&
11864              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11865                 rc = -EPROTONOSUPPORT;
11866                 goto err_free;
11867         }
11868
11869         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11870         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11871
11872         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11873         head = &bp->ntp_fltr_hash_tbl[idx];
11874         rcu_read_lock();
11875         hlist_for_each_entry_rcu(fltr, head, hash) {
11876                 if (bnxt_fltr_match(fltr, new_fltr)) {
11877                         rcu_read_unlock();
11878                         rc = 0;
11879                         goto err_free;
11880                 }
11881         }
11882         rcu_read_unlock();
11883
11884         spin_lock_bh(&bp->ntp_fltr_lock);
11885         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11886                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
11887         if (bit_id < 0) {
11888                 spin_unlock_bh(&bp->ntp_fltr_lock);
11889                 rc = -ENOMEM;
11890                 goto err_free;
11891         }
11892
11893         new_fltr->sw_id = (u16)bit_id;
11894         new_fltr->flow_id = flow_id;
11895         new_fltr->l2_fltr_idx = l2_idx;
11896         new_fltr->rxq = rxq_index;
11897         hlist_add_head_rcu(&new_fltr->hash, head);
11898         bp->ntp_fltr_count++;
11899         spin_unlock_bh(&bp->ntp_fltr_lock);
11900
11901         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11902         bnxt_queue_sp_work(bp);
11903
11904         return new_fltr->sw_id;
11905
11906 err_free:
11907         kfree(new_fltr);
11908         return rc;
11909 }
11910
11911 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11912 {
11913         int i;
11914
11915         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11916                 struct hlist_head *head;
11917                 struct hlist_node *tmp;
11918                 struct bnxt_ntuple_filter *fltr;
11919                 int rc;
11920
11921                 head = &bp->ntp_fltr_hash_tbl[i];
11922                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11923                         bool del = false;
11924
11925                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11926                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11927                                                         fltr->flow_id,
11928                                                         fltr->sw_id)) {
11929                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
11930                                                                          fltr);
11931                                         del = true;
11932                                 }
11933                         } else {
11934                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11935                                                                        fltr);
11936                                 if (rc)
11937                                         del = true;
11938                                 else
11939                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
11940                         }
11941
11942                         if (del) {
11943                                 spin_lock_bh(&bp->ntp_fltr_lock);
11944                                 hlist_del_rcu(&fltr->hash);
11945                                 bp->ntp_fltr_count--;
11946                                 spin_unlock_bh(&bp->ntp_fltr_lock);
11947                                 synchronize_rcu();
11948                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11949                                 kfree(fltr);
11950                         }
11951                 }
11952         }
11953         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11954                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
11955 }
11956
11957 #else
11958
11959 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11960 {
11961 }
11962
11963 #endif /* CONFIG_RFS_ACCEL */
11964
11965 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11966 {
11967         struct bnxt *bp = netdev_priv(netdev);
11968         struct udp_tunnel_info ti;
11969         unsigned int cmd;
11970
11971         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11972         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11973                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11974         else
11975                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11976
11977         if (ti.port)
11978                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11979
11980         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11981 }
11982
11983 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11984         .sync_table     = bnxt_udp_tunnel_sync,
11985         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11986                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11987         .tables         = {
11988                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11989                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11990         },
11991 };
11992
11993 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11994                                struct net_device *dev, u32 filter_mask,
11995                                int nlflags)
11996 {
11997         struct bnxt *bp = netdev_priv(dev);
11998
11999         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
12000                                        nlflags, filter_mask, NULL);
12001 }
12002
12003 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12004                                u16 flags, struct netlink_ext_ack *extack)
12005 {
12006         struct bnxt *bp = netdev_priv(dev);
12007         struct nlattr *attr, *br_spec;
12008         int rem, rc = 0;
12009
12010         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12011                 return -EOPNOTSUPP;
12012
12013         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12014         if (!br_spec)
12015                 return -EINVAL;
12016
12017         nla_for_each_nested(attr, br_spec, rem) {
12018                 u16 mode;
12019
12020                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12021                         continue;
12022
12023                 if (nla_len(attr) < sizeof(mode))
12024                         return -EINVAL;
12025
12026                 mode = nla_get_u16(attr);
12027                 if (mode == bp->br_mode)
12028                         break;
12029
12030                 rc = bnxt_hwrm_set_br_mode(bp, mode);
12031                 if (!rc)
12032                         bp->br_mode = mode;
12033                 break;
12034         }
12035         return rc;
12036 }
12037
12038 int bnxt_get_port_parent_id(struct net_device *dev,
12039                             struct netdev_phys_item_id *ppid)
12040 {
12041         struct bnxt *bp = netdev_priv(dev);
12042
12043         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12044                 return -EOPNOTSUPP;
12045
12046         /* The PF and it's VF-reps only support the switchdev framework */
12047         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12048                 return -EOPNOTSUPP;
12049
12050         ppid->id_len = sizeof(bp->dsn);
12051         memcpy(ppid->id, bp->dsn, ppid->id_len);
12052
12053         return 0;
12054 }
12055
12056 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12057 {
12058         struct bnxt *bp = netdev_priv(dev);
12059
12060         return &bp->dl_port;
12061 }
12062
12063 static const struct net_device_ops bnxt_netdev_ops = {
12064         .ndo_open               = bnxt_open,
12065         .ndo_start_xmit         = bnxt_start_xmit,
12066         .ndo_stop               = bnxt_close,
12067         .ndo_get_stats64        = bnxt_get_stats64,
12068         .ndo_set_rx_mode        = bnxt_set_rx_mode,
12069         .ndo_do_ioctl           = bnxt_ioctl,
12070         .ndo_validate_addr      = eth_validate_addr,
12071         .ndo_set_mac_address    = bnxt_change_mac_addr,
12072         .ndo_change_mtu         = bnxt_change_mtu,
12073         .ndo_fix_features       = bnxt_fix_features,
12074         .ndo_set_features       = bnxt_set_features,
12075         .ndo_tx_timeout         = bnxt_tx_timeout,
12076 #ifdef CONFIG_BNXT_SRIOV
12077         .ndo_get_vf_config      = bnxt_get_vf_config,
12078         .ndo_set_vf_mac         = bnxt_set_vf_mac,
12079         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
12080         .ndo_set_vf_rate        = bnxt_set_vf_bw,
12081         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
12082         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
12083         .ndo_set_vf_trust       = bnxt_set_vf_trust,
12084 #endif
12085         .ndo_setup_tc           = bnxt_setup_tc,
12086 #ifdef CONFIG_RFS_ACCEL
12087         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
12088 #endif
12089         .ndo_udp_tunnel_add     = udp_tunnel_nic_add_port,
12090         .ndo_udp_tunnel_del     = udp_tunnel_nic_del_port,
12091         .ndo_bpf                = bnxt_xdp,
12092         .ndo_xdp_xmit           = bnxt_xdp_xmit,
12093         .ndo_bridge_getlink     = bnxt_bridge_getlink,
12094         .ndo_bridge_setlink     = bnxt_bridge_setlink,
12095         .ndo_get_devlink_port   = bnxt_get_devlink_port,
12096 };
12097
12098 static void bnxt_remove_one(struct pci_dev *pdev)
12099 {
12100         struct net_device *dev = pci_get_drvdata(pdev);
12101         struct bnxt *bp = netdev_priv(dev);
12102
12103         if (BNXT_PF(bp))
12104                 bnxt_sriov_disable(bp);
12105
12106         if (BNXT_PF(bp))
12107                 devlink_port_type_clear(&bp->dl_port);
12108         pci_disable_pcie_error_reporting(pdev);
12109         unregister_netdev(dev);
12110         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12111         /* Flush any pending tasks */
12112         cancel_work_sync(&bp->sp_task);
12113         cancel_delayed_work_sync(&bp->fw_reset_task);
12114         bp->sp_event = 0;
12115
12116         bnxt_dl_fw_reporters_destroy(bp, true);
12117         bnxt_dl_unregister(bp);
12118         bnxt_shutdown_tc(bp);
12119
12120         bnxt_clear_int_mode(bp);
12121         bnxt_hwrm_func_drv_unrgtr(bp);
12122         bnxt_free_hwrm_resources(bp);
12123         bnxt_free_hwrm_short_cmd_req(bp);
12124         bnxt_ethtool_free(bp);
12125         bnxt_dcb_free(bp);
12126         kfree(bp->edev);
12127         bp->edev = NULL;
12128         kfree(bp->fw_health);
12129         bp->fw_health = NULL;
12130         bnxt_cleanup_pci(bp);
12131         bnxt_free_ctx_mem(bp);
12132         kfree(bp->ctx);
12133         bp->ctx = NULL;
12134         kfree(bp->rss_indir_tbl);
12135         bp->rss_indir_tbl = NULL;
12136         bnxt_free_port_stats(bp);
12137         free_netdev(dev);
12138 }
12139
12140 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12141 {
12142         int rc = 0;
12143         struct bnxt_link_info *link_info = &bp->link_info;
12144
12145         rc = bnxt_hwrm_phy_qcaps(bp);
12146         if (rc) {
12147                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12148                            rc);
12149                 return rc;
12150         }
12151         if (!fw_dflt)
12152                 return 0;
12153
12154         rc = bnxt_update_link(bp, false);
12155         if (rc) {
12156                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12157                            rc);
12158                 return rc;
12159         }
12160
12161         /* Older firmware does not have supported_auto_speeds, so assume
12162          * that all supported speeds can be autonegotiated.
12163          */
12164         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12165                 link_info->support_auto_speeds = link_info->support_speeds;
12166
12167         bnxt_init_ethtool_link_settings(bp);
12168         return 0;
12169 }
12170
12171 static int bnxt_get_max_irq(struct pci_dev *pdev)
12172 {
12173         u16 ctrl;
12174
12175         if (!pdev->msix_cap)
12176                 return 1;
12177
12178         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12179         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12180 }
12181
12182 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12183                                 int *max_cp)
12184 {
12185         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12186         int max_ring_grps = 0, max_irq;
12187
12188         *max_tx = hw_resc->max_tx_rings;
12189         *max_rx = hw_resc->max_rx_rings;
12190         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12191         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12192                         bnxt_get_ulp_msix_num(bp),
12193                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12194         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12195                 *max_cp = min_t(int, *max_cp, max_irq);
12196         max_ring_grps = hw_resc->max_hw_ring_grps;
12197         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12198                 *max_cp -= 1;
12199                 *max_rx -= 2;
12200         }
12201         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12202                 *max_rx >>= 1;
12203         if (bp->flags & BNXT_FLAG_CHIP_P5) {
12204                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12205                 /* On P5 chips, max_cp output param should be available NQs */
12206                 *max_cp = max_irq;
12207         }
12208         *max_rx = min_t(int, *max_rx, max_ring_grps);
12209 }
12210
12211 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12212 {
12213         int rx, tx, cp;
12214
12215         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
12216         *max_rx = rx;
12217         *max_tx = tx;
12218         if (!rx || !tx || !cp)
12219                 return -ENOMEM;
12220
12221         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12222 }
12223
12224 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12225                                bool shared)
12226 {
12227         int rc;
12228
12229         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12230         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12231                 /* Not enough rings, try disabling agg rings. */
12232                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12233                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12234                 if (rc) {
12235                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
12236                         bp->flags |= BNXT_FLAG_AGG_RINGS;
12237                         return rc;
12238                 }
12239                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12240                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12241                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12242                 bnxt_set_ring_params(bp);
12243         }
12244
12245         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12246                 int max_cp, max_stat, max_irq;
12247
12248                 /* Reserve minimum resources for RoCE */
12249                 max_cp = bnxt_get_max_func_cp_rings(bp);
12250                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
12251                 max_irq = bnxt_get_max_func_irqs(bp);
12252                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12253                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12254                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12255                         return 0;
12256
12257                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12258                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12259                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12260                 max_cp = min_t(int, max_cp, max_irq);
12261                 max_cp = min_t(int, max_cp, max_stat);
12262                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12263                 if (rc)
12264                         rc = 0;
12265         }
12266         return rc;
12267 }
12268
12269 /* In initial default shared ring setting, each shared ring must have a
12270  * RX/TX ring pair.
12271  */
12272 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12273 {
12274         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12275         bp->rx_nr_rings = bp->cp_nr_rings;
12276         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12277         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12278 }
12279
12280 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12281 {
12282         int dflt_rings, max_rx_rings, max_tx_rings, rc;
12283
12284         if (!bnxt_can_reserve_rings(bp))
12285                 return 0;
12286
12287         if (sh)
12288                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12289         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12290         /* Reduce default rings on multi-port cards so that total default
12291          * rings do not exceed CPU count.
12292          */
12293         if (bp->port_count > 1) {
12294                 int max_rings =
12295                         max_t(int, num_online_cpus() / bp->port_count, 1);
12296
12297                 dflt_rings = min_t(int, dflt_rings, max_rings);
12298         }
12299         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12300         if (rc)
12301                 return rc;
12302         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12303         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12304         if (sh)
12305                 bnxt_trim_dflt_sh_rings(bp);
12306         else
12307                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12308         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12309
12310         rc = __bnxt_reserve_rings(bp);
12311         if (rc)
12312                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12313         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12314         if (sh)
12315                 bnxt_trim_dflt_sh_rings(bp);
12316
12317         /* Rings may have been trimmed, re-reserve the trimmed rings. */
12318         if (bnxt_need_reserve_rings(bp)) {
12319                 rc = __bnxt_reserve_rings(bp);
12320                 if (rc)
12321                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12322                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12323         }
12324         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12325                 bp->rx_nr_rings++;
12326                 bp->cp_nr_rings++;
12327         }
12328         if (rc) {
12329                 bp->tx_nr_rings = 0;
12330                 bp->rx_nr_rings = 0;
12331         }
12332         return rc;
12333 }
12334
12335 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12336 {
12337         int rc;
12338
12339         if (bp->tx_nr_rings)
12340                 return 0;
12341
12342         bnxt_ulp_irq_stop(bp);
12343         bnxt_clear_int_mode(bp);
12344         rc = bnxt_set_dflt_rings(bp, true);
12345         if (rc) {
12346                 netdev_err(bp->dev, "Not enough rings available.\n");
12347                 goto init_dflt_ring_err;
12348         }
12349         rc = bnxt_init_int_mode(bp);
12350         if (rc)
12351                 goto init_dflt_ring_err;
12352
12353         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12354         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12355                 bp->flags |= BNXT_FLAG_RFS;
12356                 bp->dev->features |= NETIF_F_NTUPLE;
12357         }
12358 init_dflt_ring_err:
12359         bnxt_ulp_irq_restart(bp, rc);
12360         return rc;
12361 }
12362
12363 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12364 {
12365         int rc;
12366
12367         ASSERT_RTNL();
12368         bnxt_hwrm_func_qcaps(bp);
12369
12370         if (netif_running(bp->dev))
12371                 __bnxt_close_nic(bp, true, false);
12372
12373         bnxt_ulp_irq_stop(bp);
12374         bnxt_clear_int_mode(bp);
12375         rc = bnxt_init_int_mode(bp);
12376         bnxt_ulp_irq_restart(bp, rc);
12377
12378         if (netif_running(bp->dev)) {
12379                 if (rc)
12380                         dev_close(bp->dev);
12381                 else
12382                         rc = bnxt_open_nic(bp, true, false);
12383         }
12384
12385         return rc;
12386 }
12387
12388 static int bnxt_init_mac_addr(struct bnxt *bp)
12389 {
12390         int rc = 0;
12391
12392         if (BNXT_PF(bp)) {
12393                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12394         } else {
12395 #ifdef CONFIG_BNXT_SRIOV
12396                 struct bnxt_vf_info *vf = &bp->vf;
12397                 bool strict_approval = true;
12398
12399                 if (is_valid_ether_addr(vf->mac_addr)) {
12400                         /* overwrite netdev dev_addr with admin VF MAC */
12401                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12402                         /* Older PF driver or firmware may not approve this
12403                          * correctly.
12404                          */
12405                         strict_approval = false;
12406                 } else {
12407                         eth_hw_addr_random(bp->dev);
12408                 }
12409                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12410 #endif
12411         }
12412         return rc;
12413 }
12414
12415 #define BNXT_VPD_LEN    512
12416 static void bnxt_vpd_read_info(struct bnxt *bp)
12417 {
12418         struct pci_dev *pdev = bp->pdev;
12419         int i, len, pos, ro_size, size;
12420         ssize_t vpd_size;
12421         u8 *vpd_data;
12422
12423         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12424         if (!vpd_data)
12425                 return;
12426
12427         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12428         if (vpd_size <= 0) {
12429                 netdev_err(bp->dev, "Unable to read VPD\n");
12430                 goto exit;
12431         }
12432
12433         i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12434         if (i < 0) {
12435                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12436                 goto exit;
12437         }
12438
12439         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12440         i += PCI_VPD_LRDT_TAG_SIZE;
12441         if (i + ro_size > vpd_size)
12442                 goto exit;
12443
12444         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12445                                         PCI_VPD_RO_KEYWORD_PARTNO);
12446         if (pos < 0)
12447                 goto read_sn;
12448
12449         len = pci_vpd_info_field_size(&vpd_data[pos]);
12450         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12451         if (len + pos > vpd_size)
12452                 goto read_sn;
12453
12454         size = min(len, BNXT_VPD_FLD_LEN - 1);
12455         memcpy(bp->board_partno, &vpd_data[pos], size);
12456
12457 read_sn:
12458         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12459                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12460         if (pos < 0)
12461                 goto exit;
12462
12463         len = pci_vpd_info_field_size(&vpd_data[pos]);
12464         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12465         if (len + pos > vpd_size)
12466                 goto exit;
12467
12468         size = min(len, BNXT_VPD_FLD_LEN - 1);
12469         memcpy(bp->board_serialno, &vpd_data[pos], size);
12470 exit:
12471         kfree(vpd_data);
12472 }
12473
12474 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12475 {
12476         struct pci_dev *pdev = bp->pdev;
12477         u64 qword;
12478
12479         qword = pci_get_dsn(pdev);
12480         if (!qword) {
12481                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12482                 return -EOPNOTSUPP;
12483         }
12484
12485         put_unaligned_le64(qword, dsn);
12486
12487         bp->flags |= BNXT_FLAG_DSN_VALID;
12488         return 0;
12489 }
12490
12491 static int bnxt_map_db_bar(struct bnxt *bp)
12492 {
12493         if (!bp->db_size)
12494                 return -ENODEV;
12495         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12496         if (!bp->bar1)
12497                 return -ENOMEM;
12498         return 0;
12499 }
12500
12501 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12502 {
12503         struct net_device *dev;
12504         struct bnxt *bp;
12505         int rc, max_irqs;
12506
12507         if (pci_is_bridge(pdev))
12508                 return -ENODEV;
12509
12510         /* Clear any pending DMA transactions from crash kernel
12511          * while loading driver in capture kernel.
12512          */
12513         if (is_kdump_kernel()) {
12514                 pci_clear_master(pdev);
12515                 pcie_flr(pdev);
12516         }
12517
12518         max_irqs = bnxt_get_max_irq(pdev);
12519         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12520         if (!dev)
12521                 return -ENOMEM;
12522
12523         bp = netdev_priv(dev);
12524         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
12525         bnxt_set_max_func_irqs(bp, max_irqs);
12526
12527         if (bnxt_vf_pciid(ent->driver_data))
12528                 bp->flags |= BNXT_FLAG_VF;
12529
12530         if (pdev->msix_cap)
12531                 bp->flags |= BNXT_FLAG_MSIX_CAP;
12532
12533         rc = bnxt_init_board(pdev, dev);
12534         if (rc < 0)
12535                 goto init_err_free;
12536
12537         dev->netdev_ops = &bnxt_netdev_ops;
12538         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12539         dev->ethtool_ops = &bnxt_ethtool_ops;
12540         pci_set_drvdata(pdev, dev);
12541
12542         if (BNXT_PF(bp))
12543                 bnxt_vpd_read_info(bp);
12544
12545         rc = bnxt_alloc_hwrm_resources(bp);
12546         if (rc)
12547                 goto init_err_pci_clean;
12548
12549         mutex_init(&bp->hwrm_cmd_lock);
12550         mutex_init(&bp->link_lock);
12551
12552         rc = bnxt_fw_init_one_p1(bp);
12553         if (rc)
12554                 goto init_err_pci_clean;
12555
12556         if (BNXT_CHIP_P5(bp)) {
12557                 bp->flags |= BNXT_FLAG_CHIP_P5;
12558                 if (BNXT_CHIP_SR2(bp))
12559                         bp->flags |= BNXT_FLAG_CHIP_SR2;
12560         }
12561
12562         rc = bnxt_alloc_rss_indir_tbl(bp);
12563         if (rc)
12564                 goto init_err_pci_clean;
12565
12566         rc = bnxt_fw_init_one_p2(bp);
12567         if (rc)
12568                 goto init_err_pci_clean;
12569
12570         rc = bnxt_map_db_bar(bp);
12571         if (rc) {
12572                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12573                         rc);
12574                 goto init_err_pci_clean;
12575         }
12576
12577         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12578                            NETIF_F_TSO | NETIF_F_TSO6 |
12579                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12580                            NETIF_F_GSO_IPXIP4 |
12581                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12582                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12583                            NETIF_F_RXCSUM | NETIF_F_GRO;
12584
12585         if (BNXT_SUPPORTS_TPA(bp))
12586                 dev->hw_features |= NETIF_F_LRO;
12587
12588         dev->hw_enc_features =
12589                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12590                         NETIF_F_TSO | NETIF_F_TSO6 |
12591                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12592                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12593                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12594         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12595
12596         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12597                                     NETIF_F_GSO_GRE_CSUM;
12598         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12599         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12600                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12601         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12602                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12603         if (BNXT_SUPPORTS_TPA(bp))
12604                 dev->hw_features |= NETIF_F_GRO_HW;
12605         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12606         if (dev->features & NETIF_F_GRO_HW)
12607                 dev->features &= ~NETIF_F_LRO;
12608         dev->priv_flags |= IFF_UNICAST_FLT;
12609
12610 #ifdef CONFIG_BNXT_SRIOV
12611         init_waitqueue_head(&bp->sriov_cfg_wait);
12612         mutex_init(&bp->sriov_lock);
12613 #endif
12614         if (BNXT_SUPPORTS_TPA(bp)) {
12615                 bp->gro_func = bnxt_gro_func_5730x;
12616                 if (BNXT_CHIP_P4(bp))
12617                         bp->gro_func = bnxt_gro_func_5731x;
12618                 else if (BNXT_CHIP_P5(bp))
12619                         bp->gro_func = bnxt_gro_func_5750x;
12620         }
12621         if (!BNXT_CHIP_P4_PLUS(bp))
12622                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12623
12624         bp->ulp_probe = bnxt_ulp_probe;
12625
12626         rc = bnxt_init_mac_addr(bp);
12627         if (rc) {
12628                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12629                 rc = -EADDRNOTAVAIL;
12630                 goto init_err_pci_clean;
12631         }
12632
12633         if (BNXT_PF(bp)) {
12634                 /* Read the adapter's DSN to use as the eswitch switch_id */
12635                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12636         }
12637
12638         /* MTU range: 60 - FW defined max */
12639         dev->min_mtu = ETH_ZLEN;
12640         dev->max_mtu = bp->max_mtu;
12641
12642         rc = bnxt_probe_phy(bp, true);
12643         if (rc)
12644                 goto init_err_pci_clean;
12645
12646         bnxt_set_rx_skb_mode(bp, false);
12647         bnxt_set_tpa_flags(bp);
12648         bnxt_set_ring_params(bp);
12649         rc = bnxt_set_dflt_rings(bp, true);
12650         if (rc) {
12651                 netdev_err(bp->dev, "Not enough rings available.\n");
12652                 rc = -ENOMEM;
12653                 goto init_err_pci_clean;
12654         }
12655
12656         bnxt_fw_init_one_p3(bp);
12657
12658         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12659                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12660
12661         rc = bnxt_init_int_mode(bp);
12662         if (rc)
12663                 goto init_err_pci_clean;
12664
12665         /* No TC has been set yet and rings may have been trimmed due to
12666          * limited MSIX, so we re-initialize the TX rings per TC.
12667          */
12668         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12669
12670         if (BNXT_PF(bp)) {
12671                 if (!bnxt_pf_wq) {
12672                         bnxt_pf_wq =
12673                                 create_singlethread_workqueue("bnxt_pf_wq");
12674                         if (!bnxt_pf_wq) {
12675                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12676                                 goto init_err_pci_clean;
12677                         }
12678                 }
12679                 rc = bnxt_init_tc(bp);
12680                 if (rc)
12681                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12682                                    rc);
12683         }
12684
12685         bnxt_dl_register(bp);
12686
12687         rc = register_netdev(dev);
12688         if (rc)
12689                 goto init_err_cleanup;
12690
12691         if (BNXT_PF(bp))
12692                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12693         bnxt_dl_fw_reporters_create(bp);
12694
12695         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12696                     board_info[ent->driver_data].name,
12697                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
12698         pcie_print_link_status(pdev);
12699
12700         pci_save_state(pdev);
12701         return 0;
12702
12703 init_err_cleanup:
12704         bnxt_dl_unregister(bp);
12705         bnxt_shutdown_tc(bp);
12706         bnxt_clear_int_mode(bp);
12707
12708 init_err_pci_clean:
12709         bnxt_hwrm_func_drv_unrgtr(bp);
12710         bnxt_free_hwrm_short_cmd_req(bp);
12711         bnxt_free_hwrm_resources(bp);
12712         kfree(bp->fw_health);
12713         bp->fw_health = NULL;
12714         bnxt_cleanup_pci(bp);
12715         bnxt_free_ctx_mem(bp);
12716         kfree(bp->ctx);
12717         bp->ctx = NULL;
12718         kfree(bp->rss_indir_tbl);
12719         bp->rss_indir_tbl = NULL;
12720
12721 init_err_free:
12722         free_netdev(dev);
12723         return rc;
12724 }
12725
12726 static void bnxt_shutdown(struct pci_dev *pdev)
12727 {
12728         struct net_device *dev = pci_get_drvdata(pdev);
12729         struct bnxt *bp;
12730
12731         if (!dev)
12732                 return;
12733
12734         rtnl_lock();
12735         bp = netdev_priv(dev);
12736         if (!bp)
12737                 goto shutdown_exit;
12738
12739         if (netif_running(dev))
12740                 dev_close(dev);
12741
12742         bnxt_ulp_shutdown(bp);
12743         bnxt_clear_int_mode(bp);
12744         pci_disable_device(pdev);
12745
12746         if (system_state == SYSTEM_POWER_OFF) {
12747                 pci_wake_from_d3(pdev, bp->wol);
12748                 pci_set_power_state(pdev, PCI_D3hot);
12749         }
12750
12751 shutdown_exit:
12752         rtnl_unlock();
12753 }
12754
12755 #ifdef CONFIG_PM_SLEEP
12756 static int bnxt_suspend(struct device *device)
12757 {
12758         struct net_device *dev = dev_get_drvdata(device);
12759         struct bnxt *bp = netdev_priv(dev);
12760         int rc = 0;
12761
12762         rtnl_lock();
12763         bnxt_ulp_stop(bp);
12764         if (netif_running(dev)) {
12765                 netif_device_detach(dev);
12766                 rc = bnxt_close(dev);
12767         }
12768         bnxt_hwrm_func_drv_unrgtr(bp);
12769         pci_disable_device(bp->pdev);
12770         bnxt_free_ctx_mem(bp);
12771         kfree(bp->ctx);
12772         bp->ctx = NULL;
12773         rtnl_unlock();
12774         return rc;
12775 }
12776
12777 static int bnxt_resume(struct device *device)
12778 {
12779         struct net_device *dev = dev_get_drvdata(device);
12780         struct bnxt *bp = netdev_priv(dev);
12781         int rc = 0;
12782
12783         rtnl_lock();
12784         rc = pci_enable_device(bp->pdev);
12785         if (rc) {
12786                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12787                            rc);
12788                 goto resume_exit;
12789         }
12790         pci_set_master(bp->pdev);
12791         if (bnxt_hwrm_ver_get(bp)) {
12792                 rc = -ENODEV;
12793                 goto resume_exit;
12794         }
12795         rc = bnxt_hwrm_func_reset(bp);
12796         if (rc) {
12797                 rc = -EBUSY;
12798                 goto resume_exit;
12799         }
12800
12801         rc = bnxt_hwrm_func_qcaps(bp);
12802         if (rc)
12803                 goto resume_exit;
12804
12805         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12806                 rc = -ENODEV;
12807                 goto resume_exit;
12808         }
12809
12810         bnxt_get_wol_settings(bp);
12811         if (netif_running(dev)) {
12812                 rc = bnxt_open(dev);
12813                 if (!rc)
12814                         netif_device_attach(dev);
12815         }
12816
12817 resume_exit:
12818         bnxt_ulp_start(bp, rc);
12819         if (!rc)
12820                 bnxt_reenable_sriov(bp);
12821         rtnl_unlock();
12822         return rc;
12823 }
12824
12825 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12826 #define BNXT_PM_OPS (&bnxt_pm_ops)
12827
12828 #else
12829
12830 #define BNXT_PM_OPS NULL
12831
12832 #endif /* CONFIG_PM_SLEEP */
12833
12834 /**
12835  * bnxt_io_error_detected - called when PCI error is detected
12836  * @pdev: Pointer to PCI device
12837  * @state: The current pci connection state
12838  *
12839  * This function is called after a PCI bus error affecting
12840  * this device has been detected.
12841  */
12842 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12843                                                pci_channel_state_t state)
12844 {
12845         struct net_device *netdev = pci_get_drvdata(pdev);
12846         struct bnxt *bp = netdev_priv(netdev);
12847
12848         netdev_info(netdev, "PCI I/O error detected\n");
12849
12850         rtnl_lock();
12851         netif_device_detach(netdev);
12852
12853         bnxt_ulp_stop(bp);
12854
12855         if (state == pci_channel_io_perm_failure) {
12856                 rtnl_unlock();
12857                 return PCI_ERS_RESULT_DISCONNECT;
12858         }
12859
12860         if (state == pci_channel_io_frozen)
12861                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
12862
12863         if (netif_running(netdev))
12864                 bnxt_close(netdev);
12865
12866         pci_disable_device(pdev);
12867         bnxt_free_ctx_mem(bp);
12868         kfree(bp->ctx);
12869         bp->ctx = NULL;
12870         rtnl_unlock();
12871
12872         /* Request a slot slot reset. */
12873         return PCI_ERS_RESULT_NEED_RESET;
12874 }
12875
12876 /**
12877  * bnxt_io_slot_reset - called after the pci bus has been reset.
12878  * @pdev: Pointer to PCI device
12879  *
12880  * Restart the card from scratch, as if from a cold-boot.
12881  * At this point, the card has exprienced a hard reset,
12882  * followed by fixups by BIOS, and has its config space
12883  * set up identically to what it was at cold boot.
12884  */
12885 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12886 {
12887         struct net_device *netdev = pci_get_drvdata(pdev);
12888         struct bnxt *bp = netdev_priv(netdev);
12889         int err = 0, off;
12890         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12891
12892         netdev_info(bp->dev, "PCI Slot Reset\n");
12893
12894         rtnl_lock();
12895
12896         if (pci_enable_device(pdev)) {
12897                 dev_err(&pdev->dev,
12898                         "Cannot re-enable PCI device after reset.\n");
12899         } else {
12900                 pci_set_master(pdev);
12901                 /* Upon fatal error, our device internal logic that latches to
12902                  * BAR value is getting reset and will restore only upon
12903                  * rewritting the BARs.
12904                  *
12905                  * As pci_restore_state() does not re-write the BARs if the
12906                  * value is same as saved value earlier, driver needs to
12907                  * write the BARs to 0 to force restore, in case of fatal error.
12908                  */
12909                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
12910                                        &bp->state)) {
12911                         for (off = PCI_BASE_ADDRESS_0;
12912                              off <= PCI_BASE_ADDRESS_5; off += 4)
12913                                 pci_write_config_dword(bp->pdev, off, 0);
12914                 }
12915                 pci_restore_state(pdev);
12916                 pci_save_state(pdev);
12917
12918                 err = bnxt_hwrm_func_reset(bp);
12919                 if (!err) {
12920                         err = bnxt_hwrm_func_qcaps(bp);
12921                         if (!err && netif_running(netdev))
12922                                 err = bnxt_open(netdev);
12923                 }
12924                 bnxt_ulp_start(bp, err);
12925                 if (!err) {
12926                         bnxt_reenable_sriov(bp);
12927                         result = PCI_ERS_RESULT_RECOVERED;
12928                 }
12929         }
12930
12931         if (result != PCI_ERS_RESULT_RECOVERED) {
12932                 if (netif_running(netdev))
12933                         dev_close(netdev);
12934                 pci_disable_device(pdev);
12935         }
12936
12937         rtnl_unlock();
12938
12939         return result;
12940 }
12941
12942 /**
12943  * bnxt_io_resume - called when traffic can start flowing again.
12944  * @pdev: Pointer to PCI device
12945  *
12946  * This callback is called when the error recovery driver tells
12947  * us that its OK to resume normal operation.
12948  */
12949 static void bnxt_io_resume(struct pci_dev *pdev)
12950 {
12951         struct net_device *netdev = pci_get_drvdata(pdev);
12952
12953         rtnl_lock();
12954
12955         netif_device_attach(netdev);
12956
12957         rtnl_unlock();
12958 }
12959
12960 static const struct pci_error_handlers bnxt_err_handler = {
12961         .error_detected = bnxt_io_error_detected,
12962         .slot_reset     = bnxt_io_slot_reset,
12963         .resume         = bnxt_io_resume
12964 };
12965
12966 static struct pci_driver bnxt_pci_driver = {
12967         .name           = DRV_MODULE_NAME,
12968         .id_table       = bnxt_pci_tbl,
12969         .probe          = bnxt_init_one,
12970         .remove         = bnxt_remove_one,
12971         .shutdown       = bnxt_shutdown,
12972         .driver.pm      = BNXT_PM_OPS,
12973         .err_handler    = &bnxt_err_handler,
12974 #if defined(CONFIG_BNXT_SRIOV)
12975         .sriov_configure = bnxt_sriov_configure,
12976 #endif
12977 };
12978
12979 static int __init bnxt_init(void)
12980 {
12981         bnxt_debug_init();
12982         return pci_register_driver(&bnxt_pci_driver);
12983 }
12984
12985 static void __exit bnxt_exit(void)
12986 {
12987         pci_unregister_driver(&bnxt_pci_driver);
12988         if (bnxt_pf_wq)
12989                 destroy_workqueue(bnxt_pf_wq);
12990         bnxt_debug_exit();
12991 }
12992
12993 module_init(bnxt_init);
12994 module_exit(bnxt_exit);