drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/gro.h>
  41 #include <net/ip.h>
  42 #include <net/tcp.h>
  43 #include <net/udp.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46 #include <net/udp_tunnel.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/prefetch.h>
  49 #include <linux/cache.h>
  50 #include <linux/log2.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58 #include <linux/align.h>
  59 #include <net/netdev_queues.h>
  60
  61 #include "bnxt_hsi.h"
  62 #include "bnxt.h"
  63 #include "bnxt_hwrm.h"
  64 #include "bnxt_ulp.h"
  65 #include "bnxt_sriov.h"
  66 #include "bnxt_ethtool.h"
  67 #include "bnxt_dcb.h"
  68 #include "bnxt_xdp.h"
  69 #include "bnxt_ptp.h"
  70 #include "bnxt_vfr.h"
  71 #include "bnxt_tc.h"
  72 #include "bnxt_devlink.h"
  73 #include "bnxt_debugfs.h"
  74
  75 #define BNXT_TX_TIMEOUT         (5 * HZ)
  76 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW | \
  77                                  NETIF_MSG_TX_ERR)
  78
  79 MODULE_LICENSE("GPL");
  80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  81
  82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  84 #define BNXT_RX_COPY_THRESH 256
  85
  86 #define BNXT_TX_PUSH_THRESH 164
  87
  88 /* indexed by enum board_idx */
  89 static const struct {
  90         char *name;
  91 } board_info[] = {
  92         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
  93         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
  94         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
  95         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
  96         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
  97         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
  98         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
  99         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 100         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 101         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 102         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 103         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 104         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 105         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 106         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 107         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 108         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 109         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 110         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 111         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 112         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 113         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 114         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 115         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 116         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 117         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 118         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 119         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 120         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 121         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 122         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 123         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 124         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 125         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 126         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 127         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 128         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 129         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 130         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 131         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 132         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 133         [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
 134         [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
 135         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 136         [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
 137 };
 138
 139 static const struct pci_device_id bnxt_pci_tbl[] = {
 140         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 141         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 142         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 143         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 144         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 145         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 146         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 147         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 148         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 149         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 150         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 151         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 152         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 153         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 154         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 155         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 156         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 157         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 158         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 159         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 160         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 161         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 162         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 163         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 164         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 165         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 166         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 167         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 168         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 170         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 171         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 173         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 174         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 175         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 176         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 177         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 178         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 180         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 185         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 186 #ifdef CONFIG_BNXT_SRIOV
 187         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 188         { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
 189         { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
 190         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 191         { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
 192         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 193         { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
 194         { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
 195         { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
 196         { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
 197         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 198         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 199         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 201         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 202         { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
 203         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 204         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 205         { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
 206         { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
 207         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 208 #endif
 209         { 0 }
 210 };
 211
 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 213
 214 static const u16 bnxt_vf_req_snif[] = {
 215         HWRM_FUNC_CFG,
 216         HWRM_FUNC_VF_CFG,
 217         HWRM_PORT_PHY_QCFG,
 218         HWRM_CFA_L2_FILTER_ALLOC,
 219 };
 220
 221 static const u16 bnxt_async_events_arr[] = {
 222         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 223         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 224         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 225         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 226         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 227         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 228         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 229         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 230         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 231         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 232         ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
 233         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 234         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 235         ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
 236         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
 237         ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
 238 };
 239
 240 static struct workqueue_struct *bnxt_pf_wq;
 241
 242 static bool bnxt_vf_pciid(enum board_idx idx)
 243 {
 244         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 245                 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
 246                 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
 247                 idx == NETXTREME_E_P5_VF_HV);
 248 }
 249
 250 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 251 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 252 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 253
 254 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 255                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 256
 257 #define BNXT_DB_CQ(db, idx)                                             \
 258         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 259
 260 #define BNXT_DB_NQ_P5(db, idx)                                          \
 261         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx),   \
 262                     (db)->doorbell)
 263
 264 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 265         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 266
 267 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 268         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\
 269                     (db)->doorbell)
 270
 271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 272 {
 273         if (bp->flags & BNXT_FLAG_CHIP_P5)
 274                 BNXT_DB_NQ_P5(db, idx);
 275         else
 276                 BNXT_DB_CQ(db, idx);
 277 }
 278
 279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 280 {
 281         if (bp->flags & BNXT_FLAG_CHIP_P5)
 282                 BNXT_DB_NQ_ARM_P5(db, idx);
 283         else
 284                 BNXT_DB_CQ_ARM(db, idx);
 285 }
 286
 287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 288 {
 289         if (bp->flags & BNXT_FLAG_CHIP_P5)
 290                 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
 291                             RING_CMP(idx), db->doorbell);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 const u16 bnxt_lhint_arr[] = {
 297         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 298         TX_BD_FLAGS_LHINT_512_TO_1023,
 299         TX_BD_FLAGS_LHINT_1024_TO_2047,
 300         TX_BD_FLAGS_LHINT_1024_TO_2047,
 301         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 302         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 303         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 304         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 305         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 306         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 307         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 308         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 309         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 310         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 311         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 312         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 313         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 314         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 315         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 316 };
 317
 318 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 319 {
 320         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 321
 322         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 323                 return 0;
 324
 325         return md_dst->u.port_info.port_id;
 326 }
 327
 328 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 329                              u16 prod)
 330 {
 331         bnxt_db_write(bp, &txr->tx_db, prod);
 332         txr->kick_pending = 0;
 333 }
 334
 335 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 336 {
 337         struct bnxt *bp = netdev_priv(dev);
 338         struct tx_bd *txbd;
 339         struct tx_bd_ext *txbd1;
 340         struct netdev_queue *txq;
 341         int i;
 342         dma_addr_t mapping;
 343         unsigned int length, pad = 0;
 344         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 345         u16 prod, last_frag;
 346         struct pci_dev *pdev = bp->pdev;
 347         struct bnxt_tx_ring_info *txr;
 348         struct bnxt_sw_tx_bd *tx_buf;
 349         __le32 lflags = 0;
 350
 351         i = skb_get_queue_mapping(skb);
 352         if (unlikely(i >= bp->tx_nr_rings)) {
 353                 dev_kfree_skb_any(skb);
 354                 dev_core_stats_tx_dropped_inc(dev);
 355                 return NETDEV_TX_OK;
 356         }
 357
 358         txq = netdev_get_tx_queue(dev, i);
 359         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 360         prod = txr->tx_prod;
 361
 362         free_size = bnxt_tx_avail(bp, txr);
 363         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 364                 /* We must have raced with NAPI cleanup */
 365                 if (net_ratelimit() && txr->kick_pending)
 366                         netif_warn(bp, tx_err, dev,
 367                                    "bnxt: ring busy w/ flush pending!\n");
 368                 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 369                                         bp->tx_wake_thresh))
 370                         return NETDEV_TX_BUSY;
 371         }
 372
 373         if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
 374                 goto tx_free;
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 401                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
 402
 403                 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
 404                     atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
 405                         if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
 406                                             &ptp->tx_hdr_off)) {
 407                                 if (vlan_tag_flags)
 408                                         ptp->tx_hdr_off += VLAN_HLEN;
 409                                 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
 410                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 411                         } else {
 412                                 atomic_inc(&bp->ptp_cfg->tx_avail);
 413                         }
 414                 }
 415         }
 416
 417         if (unlikely(skb->no_fcs))
 418                 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
 419
 420         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
 421             !lflags) {
 422                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 423                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 424                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 425                 void __iomem *db = txr->tx_db.doorbell;
 426                 void *pdata = tx_push_buf->data;
 427                 u64 *end;
 428                 int j, push_len;
 429
 430                 /* Set COAL_NOW to be ready quickly for the next push */
 431                 tx_push->tx_bd_len_flags_type =
 432                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 433                                         TX_BD_TYPE_LONG_TX_BD |
 434                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 435                                         TX_BD_FLAGS_COAL_NOW |
 436                                         TX_BD_FLAGS_PACKET_END |
 437                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 438
 439                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 440                         tx_push1->tx_bd_hsize_lflags =
 441                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 442                 else
 443                         tx_push1->tx_bd_hsize_lflags = 0;
 444
 445                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 446                 tx_push1->tx_bd_cfa_action =
 447                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 448
 449                 end = pdata + length;
 450                 end = PTR_ALIGN(end, 8) - 1;
 451                 *end = 0;
 452
 453                 skb_copy_from_linear_data(skb, pdata, len);
 454                 pdata += len;
 455                 for (j = 0; j < last_frag; j++) {
 456                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 457                         void *fptr;
 458
 459                         fptr = skb_frag_address_safe(frag);
 460                         if (!fptr)
 461                                 goto normal_tx;
 462
 463                         memcpy(pdata, fptr, skb_frag_size(frag));
 464                         pdata += skb_frag_size(frag);
 465                 }
 466
 467                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 468                 txbd->tx_bd_haddr = txr->data_mapping;
 469                 prod = NEXT_TX(prod);
 470                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 471                 memcpy(txbd, tx_push1, sizeof(*txbd));
 472                 prod = NEXT_TX(prod);
 473                 tx_push->doorbell =
 474                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 475                 WRITE_ONCE(txr->tx_prod, prod);
 476
 477                 tx_buf->is_push = 1;
 478                 netdev_tx_sent_queue(txq, skb->len);
 479                 wmb();  /* Sync is_push and byte queue before pushing data */
 480
 481                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 482                 if (push_len > 16) {
 483                         __iowrite64_copy(db, tx_push_buf, 16);
 484                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 485                                          (push_len - 16) << 1);
 486                 } else {
 487                         __iowrite64_copy(db, tx_push_buf, push_len);
 488                 }
 489
 490                 goto tx_done;
 491         }
 492
 493 normal_tx:
 494         if (length < BNXT_MIN_PKT_SIZE) {
 495                 pad = BNXT_MIN_PKT_SIZE - length;
 496                 if (skb_pad(skb, pad))
 497                         /* SKB already freed. */
 498                         goto tx_kick_pending;
 499                 length = BNXT_MIN_PKT_SIZE;
 500         }
 501
 502         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 503
 504         if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 505                 goto tx_free;
 506
 507         dma_unmap_addr_set(tx_buf, mapping, mapping);
 508         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 509                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 510
 511         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 512
 513         prod = NEXT_TX(prod);
 514         txbd1 = (struct tx_bd_ext *)
 515                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 516
 517         txbd1->tx_bd_hsize_lflags = lflags;
 518         if (skb_is_gso(skb)) {
 519                 u32 hdr_len;
 520
 521                 if (skb->encapsulation)
 522                         hdr_len = skb_inner_tcp_all_headers(skb);
 523                 else
 524                         hdr_len = skb_tcp_all_headers(skb);
 525
 526                 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
 527                                         TX_BD_FLAGS_T_IPID |
 528                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 529                 length = skb_shinfo(skb)->gso_size;
 530                 txbd1->tx_bd_mss = cpu_to_le32(length);
 531                 length += hdr_len;
 532         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 533                 txbd1->tx_bd_hsize_lflags |=
 534                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 535                 txbd1->tx_bd_mss = 0;
 536         }
 537
 538         length >>= 9;
 539         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 540                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 541                                      skb->len);
 542                 i = 0;
 543                 goto tx_dma_error;
 544         }
 545         flags |= bnxt_lhint_arr[length];
 546         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 547
 548         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 549         txbd1->tx_bd_cfa_action =
 550                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 551         for (i = 0; i < last_frag; i++) {
 552                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 553
 554                 prod = NEXT_TX(prod);
 555                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 556
 557                 len = skb_frag_size(frag);
 558                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 559                                            DMA_TO_DEVICE);
 560
 561                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 562                         goto tx_dma_error;
 563
 564                 tx_buf = &txr->tx_buf_ring[prod];
 565                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 566
 567                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 568
 569                 flags = len << TX_BD_LEN_SHIFT;
 570                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 571         }
 572
 573         flags &= ~TX_BD_LEN;
 574         txbd->tx_bd_len_flags_type =
 575                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 576                             TX_BD_FLAGS_PACKET_END);
 577
 578         netdev_tx_sent_queue(txq, skb->len);
 579
 580         skb_tx_timestamp(skb);
 581
 582         /* Sync BD data before updating doorbell */
 583         wmb();
 584
 585         prod = NEXT_TX(prod);
 586         WRITE_ONCE(txr->tx_prod, prod);
 587
 588         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 589                 bnxt_txr_db_kick(bp, txr, prod);
 590         else
 591                 txr->kick_pending = 1;
 592
 593 tx_done:
 594
 595         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 596                 if (netdev_xmit_more() && !tx_buf->is_push)
 597                         bnxt_txr_db_kick(bp, txr, prod);
 598
 599                 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 600                                    bp->tx_wake_thresh);
 601         }
 602         return NETDEV_TX_OK;
 603
 604 tx_dma_error:
 605         if (BNXT_TX_PTP_IS_SET(lflags))
 606                 atomic_inc(&bp->ptp_cfg->tx_avail);
 607
 608         last_frag = i;
 609
 610         /* start back at beginning and unmap skb */
 611         prod = txr->tx_prod;
 612         tx_buf = &txr->tx_buf_ring[prod];
 613         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 614                          skb_headlen(skb), DMA_TO_DEVICE);
 615         prod = NEXT_TX(prod);
 616
 617         /* unmap remaining mapped pages */
 618         for (i = 0; i < last_frag; i++) {
 619                 prod = NEXT_TX(prod);
 620                 tx_buf = &txr->tx_buf_ring[prod];
 621                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 622                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 623                                DMA_TO_DEVICE);
 624         }
 625
 626 tx_free:
 627         dev_kfree_skb_any(skb);
 628 tx_kick_pending:
 629         if (txr->kick_pending)
 630                 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
 631         txr->tx_buf_ring[txr->tx_prod].skb = NULL;
 632         dev_core_stats_tx_dropped_inc(dev);
 633         return NETDEV_TX_OK;
 634 }
 635
 636 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 637 {
 638         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 639         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 640         u16 cons = txr->tx_cons;
 641         struct pci_dev *pdev = bp->pdev;
 642         int i;
 643         unsigned int tx_bytes = 0;
 644
 645         for (i = 0; i < nr_pkts; i++) {
 646                 struct bnxt_sw_tx_bd *tx_buf;
 647                 struct sk_buff *skb;
 648                 int j, last;
 649
 650                 tx_buf = &txr->tx_buf_ring[cons];
 651                 cons = NEXT_TX(cons);
 652                 skb = tx_buf->skb;
 653                 tx_buf->skb = NULL;
 654
 655                 tx_bytes += skb->len;
 656
 657                 if (tx_buf->is_push) {
 658                         tx_buf->is_push = 0;
 659                         goto next_tx_int;
 660                 }
 661
 662                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 663                                  skb_headlen(skb), DMA_TO_DEVICE);
 664                 last = tx_buf->nr_frags;
 665
 666                 for (j = 0; j < last; j++) {
 667                         cons = NEXT_TX(cons);
 668                         tx_buf = &txr->tx_buf_ring[cons];
 669                         dma_unmap_page(
 670                                 &pdev->dev,
 671                                 dma_unmap_addr(tx_buf, mapping),
 672                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 673                                 DMA_TO_DEVICE);
 674                 }
 675                 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
 676                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
 677                                 /* PTP worker takes ownership of the skb */
 678                                 if (!bnxt_get_tx_ts_p5(bp, skb))
 679                                         skb = NULL;
 680                                 else
 681                                         atomic_inc(&bp->ptp_cfg->tx_avail);
 682                         }
 683                 }
 684
 685 next_tx_int:
 686                 cons = NEXT_TX(cons);
 687
 688                 dev_kfree_skb_any(skb);
 689         }
 690
 691         WRITE_ONCE(txr->tx_cons, cons);
 692
 693         __netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
 694                                    bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
 695                                    READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING);
 696 }
 697
 698 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 699                                          struct bnxt_rx_ring_info *rxr,
 700                                          gfp_t gfp)
 701 {
 702         struct device *dev = &bp->pdev->dev;
 703         struct page *page;
 704
 705         page = page_pool_dev_alloc_pages(rxr->page_pool);
 706         if (!page)
 707                 return NULL;
 708
 709         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 710                                       DMA_ATTR_WEAK_ORDERING);
 711         if (dma_mapping_error(dev, *mapping)) {
 712                 page_pool_recycle_direct(rxr->page_pool, page);
 713                 return NULL;
 714         }
 715         return page;
 716 }
 717
 718 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
 719                                        gfp_t gfp)
 720 {
 721         u8 *data;
 722         struct pci_dev *pdev = bp->pdev;
 723
 724         if (gfp == GFP_ATOMIC)
 725                 data = napi_alloc_frag(bp->rx_buf_size);
 726         else
 727                 data = netdev_alloc_frag(bp->rx_buf_size);
 728         if (!data)
 729                 return NULL;
 730
 731         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 732                                         bp->rx_buf_use_size, bp->rx_dir,
 733                                         DMA_ATTR_WEAK_ORDERING);
 734
 735         if (dma_mapping_error(&pdev->dev, *mapping)) {
 736                 skb_free_frag(data);
 737                 data = NULL;
 738         }
 739         return data;
 740 }
 741
 742 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 743                        u16 prod, gfp_t gfp)
 744 {
 745         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 746         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 747         dma_addr_t mapping;
 748
 749         if (BNXT_RX_PAGE_MODE(bp)) {
 750                 struct page *page =
 751                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 752
 753                 if (!page)
 754                         return -ENOMEM;
 755
 756                 mapping += bp->rx_dma_offset;
 757                 rx_buf->data = page;
 758                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 759         } else {
 760                 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
 761
 762                 if (!data)
 763                         return -ENOMEM;
 764
 765                 rx_buf->data = data;
 766                 rx_buf->data_ptr = data + bp->rx_offset;
 767         }
 768         rx_buf->mapping = mapping;
 769
 770         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 771         return 0;
 772 }
 773
 774 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 775 {
 776         u16 prod = rxr->rx_prod;
 777         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 778         struct rx_bd *cons_bd, *prod_bd;
 779
 780         prod_rx_buf = &rxr->rx_buf_ring[prod];
 781         cons_rx_buf = &rxr->rx_buf_ring[cons];
 782
 783         prod_rx_buf->data = data;
 784         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 785
 786         prod_rx_buf->mapping = cons_rx_buf->mapping;
 787
 788         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 789         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 790
 791         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 792 }
 793
 794 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 795 {
 796         u16 next, max = rxr->rx_agg_bmap_size;
 797
 798         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 799         if (next >= max)
 800                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 801         return next;
 802 }
 803
 804 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 805                                      struct bnxt_rx_ring_info *rxr,
 806                                      u16 prod, gfp_t gfp)
 807 {
 808         struct rx_bd *rxbd =
 809                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 810         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 811         struct pci_dev *pdev = bp->pdev;
 812         struct page *page;
 813         dma_addr_t mapping;
 814         u16 sw_prod = rxr->rx_sw_agg_prod;
 815         unsigned int offset = 0;
 816
 817         if (BNXT_RX_PAGE_MODE(bp)) {
 818                 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 819
 820                 if (!page)
 821                         return -ENOMEM;
 822
 823         } else {
 824                 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 825                         page = rxr->rx_page;
 826                         if (!page) {
 827                                 page = alloc_page(gfp);
 828                                 if (!page)
 829                                         return -ENOMEM;
 830                                 rxr->rx_page = page;
 831                                 rxr->rx_page_offset = 0;
 832                         }
 833                         offset = rxr->rx_page_offset;
 834                         rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 835                         if (rxr->rx_page_offset == PAGE_SIZE)
 836                                 rxr->rx_page = NULL;
 837                         else
 838                                 get_page(page);
 839                 } else {
 840                         page = alloc_page(gfp);
 841                         if (!page)
 842                                 return -ENOMEM;
 843                 }
 844
 845                 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 846                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
 847                                              DMA_ATTR_WEAK_ORDERING);
 848                 if (dma_mapping_error(&pdev->dev, mapping)) {
 849                         __free_page(page);
 850                         return -EIO;
 851                 }
 852         }
 853
 854         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 855                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 856
 857         __set_bit(sw_prod, rxr->rx_agg_bmap);
 858         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 859         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 860
 861         rx_agg_buf->page = page;
 862         rx_agg_buf->offset = offset;
 863         rx_agg_buf->mapping = mapping;
 864         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 865         rxbd->rx_bd_opaque = sw_prod;
 866         return 0;
 867 }
 868
 869 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 870                                        struct bnxt_cp_ring_info *cpr,
 871                                        u16 cp_cons, u16 curr)
 872 {
 873         struct rx_agg_cmp *agg;
 874
 875         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 876         agg = (struct rx_agg_cmp *)
 877                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 878         return agg;
 879 }
 880
 881 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 882                                               struct bnxt_rx_ring_info *rxr,
 883                                               u16 agg_id, u16 curr)
 884 {
 885         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 886
 887         return &tpa_info->agg_arr[curr];
 888 }
 889
 890 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 891                                    u16 start, u32 agg_bufs, bool tpa)
 892 {
 893         struct bnxt_napi *bnapi = cpr->bnapi;
 894         struct bnxt *bp = bnapi->bp;
 895         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 896         u16 prod = rxr->rx_agg_prod;
 897         u16 sw_prod = rxr->rx_sw_agg_prod;
 898         bool p5_tpa = false;
 899         u32 i;
 900
 901         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 902                 p5_tpa = true;
 903
 904         for (i = 0; i < agg_bufs; i++) {
 905                 u16 cons;
 906                 struct rx_agg_cmp *agg;
 907                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 908                 struct rx_bd *prod_bd;
 909                 struct page *page;
 910
 911                 if (p5_tpa)
 912                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 913                 else
 914                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 915                 cons = agg->rx_agg_cmp_opaque;
 916                 __clear_bit(cons, rxr->rx_agg_bmap);
 917
 918                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 919                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 920
 921                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 922                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 923                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 924
 925                 /* It is possible for sw_prod to be equal to cons, so
 926                  * set cons_rx_buf->page to NULL first.
 927                  */
 928                 page = cons_rx_buf->page;
 929                 cons_rx_buf->page = NULL;
 930                 prod_rx_buf->page = page;
 931                 prod_rx_buf->offset = cons_rx_buf->offset;
 932
 933                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 934
 935                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 936
 937                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 938                 prod_bd->rx_bd_opaque = sw_prod;
 939
 940                 prod = NEXT_RX_AGG(prod);
 941                 sw_prod = NEXT_RX_AGG(sw_prod);
 942         }
 943         rxr->rx_agg_prod = prod;
 944         rxr->rx_sw_agg_prod = sw_prod;
 945 }
 946
 947 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
 948                                               struct bnxt_rx_ring_info *rxr,
 949                                               u16 cons, void *data, u8 *data_ptr,
 950                                               dma_addr_t dma_addr,
 951                                               unsigned int offset_and_len)
 952 {
 953         unsigned int len = offset_and_len & 0xffff;
 954         struct page *page = data;
 955         u16 prod = rxr->rx_prod;
 956         struct sk_buff *skb;
 957         int err;
 958
 959         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 960         if (unlikely(err)) {
 961                 bnxt_reuse_rx_data(rxr, cons, data);
 962                 return NULL;
 963         }
 964         dma_addr -= bp->rx_dma_offset;
 965         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 966                              DMA_ATTR_WEAK_ORDERING);
 967         skb = build_skb(page_address(page), PAGE_SIZE);
 968         if (!skb) {
 969                 page_pool_recycle_direct(rxr->page_pool, page);
 970                 return NULL;
 971         }
 972         skb_mark_for_recycle(skb);
 973         skb_reserve(skb, bp->rx_dma_offset);
 974         __skb_put(skb, len);
 975
 976         return skb;
 977 }
 978
 979 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 980                                         struct bnxt_rx_ring_info *rxr,
 981                                         u16 cons, void *data, u8 *data_ptr,
 982                                         dma_addr_t dma_addr,
 983                                         unsigned int offset_and_len)
 984 {
 985         unsigned int payload = offset_and_len >> 16;
 986         unsigned int len = offset_and_len & 0xffff;
 987         skb_frag_t *frag;
 988         struct page *page = data;
 989         u16 prod = rxr->rx_prod;
 990         struct sk_buff *skb;
 991         int off, err;
 992
 993         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 994         if (unlikely(err)) {
 995                 bnxt_reuse_rx_data(rxr, cons, data);
 996                 return NULL;
 997         }
 998         dma_addr -= bp->rx_dma_offset;
 999         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
1000                              DMA_ATTR_WEAK_ORDERING);
1001
1002         if (unlikely(!payload))
1003                 payload = eth_get_headlen(bp->dev, data_ptr, len);
1004
1005         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1006         if (!skb) {
1007                 page_pool_recycle_direct(rxr->page_pool, page);
1008                 return NULL;
1009         }
1010
1011         skb_mark_for_recycle(skb);
1012         off = (void *)data_ptr - page_address(page);
1013         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
1014         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1015                payload + NET_IP_ALIGN);
1016
1017         frag = &skb_shinfo(skb)->frags[0];
1018         skb_frag_size_sub(frag, payload);
1019         skb_frag_off_add(frag, payload);
1020         skb->data_len -= payload;
1021         skb->tail += payload;
1022
1023         return skb;
1024 }
1025
1026 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1027                                    struct bnxt_rx_ring_info *rxr, u16 cons,
1028                                    void *data, u8 *data_ptr,
1029                                    dma_addr_t dma_addr,
1030                                    unsigned int offset_and_len)
1031 {
1032         u16 prod = rxr->rx_prod;
1033         struct sk_buff *skb;
1034         int err;
1035
1036         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1037         if (unlikely(err)) {
1038                 bnxt_reuse_rx_data(rxr, cons, data);
1039                 return NULL;
1040         }
1041
1042         skb = build_skb(data, bp->rx_buf_size);
1043         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1044                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1045         if (!skb) {
1046                 skb_free_frag(data);
1047                 return NULL;
1048         }
1049
1050         skb_reserve(skb, bp->rx_offset);
1051         skb_put(skb, offset_and_len & 0xffff);
1052         return skb;
1053 }
1054
1055 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1056                                struct bnxt_cp_ring_info *cpr,
1057                                struct skb_shared_info *shinfo,
1058                                u16 idx, u32 agg_bufs, bool tpa,
1059                                struct xdp_buff *xdp)
1060 {
1061         struct bnxt_napi *bnapi = cpr->bnapi;
1062         struct pci_dev *pdev = bp->pdev;
1063         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1064         u16 prod = rxr->rx_agg_prod;
1065         u32 i, total_frag_len = 0;
1066         bool p5_tpa = false;
1067
1068         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1069                 p5_tpa = true;
1070
1071         for (i = 0; i < agg_bufs; i++) {
1072                 skb_frag_t *frag = &shinfo->frags[i];
1073                 u16 cons, frag_len;
1074                 struct rx_agg_cmp *agg;
1075                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1076                 struct page *page;
1077                 dma_addr_t mapping;
1078
1079                 if (p5_tpa)
1080                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1081                 else
1082                         agg = bnxt_get_agg(bp, cpr, idx, i);
1083                 cons = agg->rx_agg_cmp_opaque;
1084                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1085                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1086
1087                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1088                 skb_frag_off_set(frag, cons_rx_buf->offset);
1089                 skb_frag_size_set(frag, frag_len);
1090                 __skb_frag_set_page(frag, cons_rx_buf->page);
1091                 shinfo->nr_frags = i + 1;
1092                 __clear_bit(cons, rxr->rx_agg_bmap);
1093
1094                 /* It is possible for bnxt_alloc_rx_page() to allocate
1095                  * a sw_prod index that equals the cons index, so we
1096                  * need to clear the cons entry now.
1097                  */
1098                 mapping = cons_rx_buf->mapping;
1099                 page = cons_rx_buf->page;
1100                 cons_rx_buf->page = NULL;
1101
1102                 if (xdp && page_is_pfmemalloc(page))
1103                         xdp_buff_set_frag_pfmemalloc(xdp);
1104
1105                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1106                         unsigned int nr_frags;
1107
1108                         nr_frags = --shinfo->nr_frags;
1109                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1110                         cons_rx_buf->page = page;
1111
1112                         /* Update prod since possibly some pages have been
1113                          * allocated already.
1114                          */
1115                         rxr->rx_agg_prod = prod;
1116                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1117                         return 0;
1118                 }
1119
1120                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1121                                      bp->rx_dir,
1122                                      DMA_ATTR_WEAK_ORDERING);
1123
1124                 total_frag_len += frag_len;
1125                 prod = NEXT_RX_AGG(prod);
1126         }
1127         rxr->rx_agg_prod = prod;
1128         return total_frag_len;
1129 }
1130
1131 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1132                                              struct bnxt_cp_ring_info *cpr,
1133                                              struct sk_buff *skb, u16 idx,
1134                                              u32 agg_bufs, bool tpa)
1135 {
1136         struct skb_shared_info *shinfo = skb_shinfo(skb);
1137         u32 total_frag_len = 0;
1138
1139         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1140                                              agg_bufs, tpa, NULL);
1141         if (!total_frag_len) {
1142                 dev_kfree_skb(skb);
1143                 return NULL;
1144         }
1145
1146         skb->data_len += total_frag_len;
1147         skb->len += total_frag_len;
1148         skb->truesize += PAGE_SIZE * agg_bufs;
1149         return skb;
1150 }
1151
1152 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1153                                  struct bnxt_cp_ring_info *cpr,
1154                                  struct xdp_buff *xdp, u16 idx,
1155                                  u32 agg_bufs, bool tpa)
1156 {
1157         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1158         u32 total_frag_len = 0;
1159
1160         if (!xdp_buff_has_frags(xdp))
1161                 shinfo->nr_frags = 0;
1162
1163         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1164                                              idx, agg_bufs, tpa, xdp);
1165         if (total_frag_len) {
1166                 xdp_buff_set_frags_flag(xdp);
1167                 shinfo->nr_frags = agg_bufs;
1168                 shinfo->xdp_frags_size = total_frag_len;
1169         }
1170         return total_frag_len;
1171 }
1172
1173 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1174                                u8 agg_bufs, u32 *raw_cons)
1175 {
1176         u16 last;
1177         struct rx_agg_cmp *agg;
1178
1179         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1180         last = RING_CMP(*raw_cons);
1181         agg = (struct rx_agg_cmp *)
1182                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1183         return RX_AGG_CMP_VALID(agg, *raw_cons);
1184 }
1185
1186 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1187                                             unsigned int len,
1188                                             dma_addr_t mapping)
1189 {
1190         struct bnxt *bp = bnapi->bp;
1191         struct pci_dev *pdev = bp->pdev;
1192         struct sk_buff *skb;
1193
1194         skb = napi_alloc_skb(&bnapi->napi, len);
1195         if (!skb)
1196                 return NULL;
1197
1198         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1199                                 bp->rx_dir);
1200
1201         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1202                len + NET_IP_ALIGN);
1203
1204         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1205                                    bp->rx_dir);
1206
1207         skb_put(skb, len);
1208         return skb;
1209 }
1210
1211 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1212                            u32 *raw_cons, void *cmp)
1213 {
1214         struct rx_cmp *rxcmp = cmp;
1215         u32 tmp_raw_cons = *raw_cons;
1216         u8 cmp_type, agg_bufs = 0;
1217
1218         cmp_type = RX_CMP_TYPE(rxcmp);
1219
1220         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1221                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1222                             RX_CMP_AGG_BUFS) >>
1223                            RX_CMP_AGG_BUFS_SHIFT;
1224         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1225                 struct rx_tpa_end_cmp *tpa_end = cmp;
1226
1227                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1228                         return 0;
1229
1230                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1231         }
1232
1233         if (agg_bufs) {
1234                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1235                         return -EBUSY;
1236         }
1237         *raw_cons = tmp_raw_cons;
1238         return 0;
1239 }
1240
1241 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1242 {
1243         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1244                 return;
1245
1246         if (BNXT_PF(bp))
1247                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1248         else
1249                 schedule_delayed_work(&bp->fw_reset_task, delay);
1250 }
1251
1252 static void bnxt_queue_sp_work(struct bnxt *bp)
1253 {
1254         if (BNXT_PF(bp))
1255                 queue_work(bnxt_pf_wq, &bp->sp_task);
1256         else
1257                 schedule_work(&bp->sp_task);
1258 }
1259
1260 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1261 {
1262         if (!rxr->bnapi->in_reset) {
1263                 rxr->bnapi->in_reset = true;
1264                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1265                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1266                 else
1267                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1268                 bnxt_queue_sp_work(bp);
1269         }
1270         rxr->rx_next_cons = 0xffff;
1271 }
1272
1273 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1274 {
1275         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1276         u16 idx = agg_id & MAX_TPA_P5_MASK;
1277
1278         if (test_bit(idx, map->agg_idx_bmap))
1279                 idx = find_first_zero_bit(map->agg_idx_bmap,
1280                                           BNXT_AGG_IDX_BMAP_SIZE);
1281         __set_bit(idx, map->agg_idx_bmap);
1282         map->agg_id_tbl[agg_id] = idx;
1283         return idx;
1284 }
1285
1286 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1287 {
1288         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1289
1290         __clear_bit(idx, map->agg_idx_bmap);
1291 }
1292
1293 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1294 {
1295         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1296
1297         return map->agg_id_tbl[agg_id];
1298 }
1299
1300 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1301                            struct rx_tpa_start_cmp *tpa_start,
1302                            struct rx_tpa_start_cmp_ext *tpa_start1)
1303 {
1304         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1305         struct bnxt_tpa_info *tpa_info;
1306         u16 cons, prod, agg_id;
1307         struct rx_bd *prod_bd;
1308         dma_addr_t mapping;
1309
1310         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1311                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1312                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1313         } else {
1314                 agg_id = TPA_START_AGG_ID(tpa_start);
1315         }
1316         cons = tpa_start->rx_tpa_start_cmp_opaque;
1317         prod = rxr->rx_prod;
1318         cons_rx_buf = &rxr->rx_buf_ring[cons];
1319         prod_rx_buf = &rxr->rx_buf_ring[prod];
1320         tpa_info = &rxr->rx_tpa[agg_id];
1321
1322         if (unlikely(cons != rxr->rx_next_cons ||
1323                      TPA_START_ERROR(tpa_start))) {
1324                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1325                             cons, rxr->rx_next_cons,
1326                             TPA_START_ERROR_CODE(tpa_start1));
1327                 bnxt_sched_reset(bp, rxr);
1328                 return;
1329         }
1330         /* Store cfa_code in tpa_info to use in tpa_end
1331          * completion processing.
1332          */
1333         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1334         prod_rx_buf->data = tpa_info->data;
1335         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1336
1337         mapping = tpa_info->mapping;
1338         prod_rx_buf->mapping = mapping;
1339
1340         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1341
1342         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1343
1344         tpa_info->data = cons_rx_buf->data;
1345         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1346         cons_rx_buf->data = NULL;
1347         tpa_info->mapping = cons_rx_buf->mapping;
1348
1349         tpa_info->len =
1350                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1351                                 RX_TPA_START_CMP_LEN_SHIFT;
1352         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1353                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1354
1355                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1356                 tpa_info->gso_type = SKB_GSO_TCPV4;
1357                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1358                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1359                         tpa_info->gso_type = SKB_GSO_TCPV6;
1360                 tpa_info->rss_hash =
1361                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1362         } else {
1363                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1364                 tpa_info->gso_type = 0;
1365                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1366         }
1367         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1368         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1369         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1370         tpa_info->agg_count = 0;
1371
1372         rxr->rx_prod = NEXT_RX(prod);
1373         cons = NEXT_RX(cons);
1374         rxr->rx_next_cons = NEXT_RX(cons);
1375         cons_rx_buf = &rxr->rx_buf_ring[cons];
1376
1377         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1378         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1379         cons_rx_buf->data = NULL;
1380 }
1381
1382 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1383 {
1384         if (agg_bufs)
1385                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1386 }
1387
1388 #ifdef CONFIG_INET
1389 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1390 {
1391         struct udphdr *uh = NULL;
1392
1393         if (ip_proto == htons(ETH_P_IP)) {
1394                 struct iphdr *iph = (struct iphdr *)skb->data;
1395
1396                 if (iph->protocol == IPPROTO_UDP)
1397                         uh = (struct udphdr *)(iph + 1);
1398         } else {
1399                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1400
1401                 if (iph->nexthdr == IPPROTO_UDP)
1402                         uh = (struct udphdr *)(iph + 1);
1403         }
1404         if (uh) {
1405                 if (uh->check)
1406                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1407                 else
1408                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1409         }
1410 }
1411 #endif
1412
1413 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1414                                            int payload_off, int tcp_ts,
1415                                            struct sk_buff *skb)
1416 {
1417 #ifdef CONFIG_INET
1418         struct tcphdr *th;
1419         int len, nw_off;
1420         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1421         u32 hdr_info = tpa_info->hdr_info;
1422         bool loopback = false;
1423
1424         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1425         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1426         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1427
1428         /* If the packet is an internal loopback packet, the offsets will
1429          * have an extra 4 bytes.
1430          */
1431         if (inner_mac_off == 4) {
1432                 loopback = true;
1433         } else if (inner_mac_off > 4) {
1434                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1435                                             ETH_HLEN - 2));
1436
1437                 /* We only support inner iPv4/ipv6.  If we don't see the
1438                  * correct protocol ID, it must be a loopback packet where
1439                  * the offsets are off by 4.
1440                  */
1441                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1442                         loopback = true;
1443         }
1444         if (loopback) {
1445                 /* internal loopback packet, subtract all offsets by 4 */
1446                 inner_ip_off -= 4;
1447                 inner_mac_off -= 4;
1448                 outer_ip_off -= 4;
1449         }
1450
1451         nw_off = inner_ip_off - ETH_HLEN;
1452         skb_set_network_header(skb, nw_off);
1453         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1454                 struct ipv6hdr *iph = ipv6_hdr(skb);
1455
1456                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1457                 len = skb->len - skb_transport_offset(skb);
1458                 th = tcp_hdr(skb);
1459                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1460         } else {
1461                 struct iphdr *iph = ip_hdr(skb);
1462
1463                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1464                 len = skb->len - skb_transport_offset(skb);
1465                 th = tcp_hdr(skb);
1466                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1467         }
1468
1469         if (inner_mac_off) { /* tunnel */
1470                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1471                                             ETH_HLEN - 2));
1472
1473                 bnxt_gro_tunnel(skb, proto);
1474         }
1475 #endif
1476         return skb;
1477 }
1478
1479 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1480                                            int payload_off, int tcp_ts,
1481                                            struct sk_buff *skb)
1482 {
1483 #ifdef CONFIG_INET
1484         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1485         u32 hdr_info = tpa_info->hdr_info;
1486         int iphdr_len, nw_off;
1487
1488         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1489         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1490         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1491
1492         nw_off = inner_ip_off - ETH_HLEN;
1493         skb_set_network_header(skb, nw_off);
1494         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1495                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1496         skb_set_transport_header(skb, nw_off + iphdr_len);
1497
1498         if (inner_mac_off) { /* tunnel */
1499                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1500                                             ETH_HLEN - 2));
1501
1502                 bnxt_gro_tunnel(skb, proto);
1503         }
1504 #endif
1505         return skb;
1506 }
1507
1508 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1509 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1510
1511 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1512                                            int payload_off, int tcp_ts,
1513                                            struct sk_buff *skb)
1514 {
1515 #ifdef CONFIG_INET
1516         struct tcphdr *th;
1517         int len, nw_off, tcp_opt_len = 0;
1518
1519         if (tcp_ts)
1520                 tcp_opt_len = 12;
1521
1522         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1523                 struct iphdr *iph;
1524
1525                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1526                          ETH_HLEN;
1527                 skb_set_network_header(skb, nw_off);
1528                 iph = ip_hdr(skb);
1529                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1530                 len = skb->len - skb_transport_offset(skb);
1531                 th = tcp_hdr(skb);
1532                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1533         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1534                 struct ipv6hdr *iph;
1535
1536                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1537                          ETH_HLEN;
1538                 skb_set_network_header(skb, nw_off);
1539                 iph = ipv6_hdr(skb);
1540                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1541                 len = skb->len - skb_transport_offset(skb);
1542                 th = tcp_hdr(skb);
1543                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1544         } else {
1545                 dev_kfree_skb_any(skb);
1546                 return NULL;
1547         }
1548
1549         if (nw_off) /* tunnel */
1550                 bnxt_gro_tunnel(skb, skb->protocol);
1551 #endif
1552         return skb;
1553 }
1554
1555 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1556                                            struct bnxt_tpa_info *tpa_info,
1557                                            struct rx_tpa_end_cmp *tpa_end,
1558                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1559                                            struct sk_buff *skb)
1560 {
1561 #ifdef CONFIG_INET
1562         int payload_off;
1563         u16 segs;
1564
1565         segs = TPA_END_TPA_SEGS(tpa_end);
1566         if (segs == 1)
1567                 return skb;
1568
1569         NAPI_GRO_CB(skb)->count = segs;
1570         skb_shinfo(skb)->gso_size =
1571                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1572         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1573         if (bp->flags & BNXT_FLAG_CHIP_P5)
1574                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1575         else
1576                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1577         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1578         if (likely(skb))
1579                 tcp_gro_complete(skb);
1580 #endif
1581         return skb;
1582 }
1583
1584 /* Given the cfa_code of a received packet determine which
1585  * netdev (vf-rep or PF) the packet is destined to.
1586  */
1587 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1588 {
1589         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1590
1591         /* if vf-rep dev is NULL, the must belongs to the PF */
1592         return dev ? dev : bp->dev;
1593 }
1594
1595 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1596                                            struct bnxt_cp_ring_info *cpr,
1597                                            u32 *raw_cons,
1598                                            struct rx_tpa_end_cmp *tpa_end,
1599                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1600                                            u8 *event)
1601 {
1602         struct bnxt_napi *bnapi = cpr->bnapi;
1603         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1604         u8 *data_ptr, agg_bufs;
1605         unsigned int len;
1606         struct bnxt_tpa_info *tpa_info;
1607         dma_addr_t mapping;
1608         struct sk_buff *skb;
1609         u16 idx = 0, agg_id;
1610         void *data;
1611         bool gro;
1612
1613         if (unlikely(bnapi->in_reset)) {
1614                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1615
1616                 if (rc < 0)
1617                         return ERR_PTR(-EBUSY);
1618                 return NULL;
1619         }
1620
1621         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1622                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1623                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1624                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1625                 tpa_info = &rxr->rx_tpa[agg_id];
1626                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1627                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1628                                     agg_bufs, tpa_info->agg_count);
1629                         agg_bufs = tpa_info->agg_count;
1630                 }
1631                 tpa_info->agg_count = 0;
1632                 *event |= BNXT_AGG_EVENT;
1633                 bnxt_free_agg_idx(rxr, agg_id);
1634                 idx = agg_id;
1635                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1636         } else {
1637                 agg_id = TPA_END_AGG_ID(tpa_end);
1638                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1639                 tpa_info = &rxr->rx_tpa[agg_id];
1640                 idx = RING_CMP(*raw_cons);
1641                 if (agg_bufs) {
1642                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1643                                 return ERR_PTR(-EBUSY);
1644
1645                         *event |= BNXT_AGG_EVENT;
1646                         idx = NEXT_CMP(idx);
1647                 }
1648                 gro = !!TPA_END_GRO(tpa_end);
1649         }
1650         data = tpa_info->data;
1651         data_ptr = tpa_info->data_ptr;
1652         prefetch(data_ptr);
1653         len = tpa_info->len;
1654         mapping = tpa_info->mapping;
1655
1656         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1657                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1658                 if (agg_bufs > MAX_SKB_FRAGS)
1659                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1660                                     agg_bufs, (int)MAX_SKB_FRAGS);
1661                 return NULL;
1662         }
1663
1664         if (len <= bp->rx_copy_thresh) {
1665                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1666                 if (!skb) {
1667                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1668                         cpr->sw_stats.rx.rx_oom_discards += 1;
1669                         return NULL;
1670                 }
1671         } else {
1672                 u8 *new_data;
1673                 dma_addr_t new_mapping;
1674
1675                 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1676                 if (!new_data) {
1677                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1678                         cpr->sw_stats.rx.rx_oom_discards += 1;
1679                         return NULL;
1680                 }
1681
1682                 tpa_info->data = new_data;
1683                 tpa_info->data_ptr = new_data + bp->rx_offset;
1684                 tpa_info->mapping = new_mapping;
1685
1686                 skb = build_skb(data, bp->rx_buf_size);
1687                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1688                                        bp->rx_buf_use_size, bp->rx_dir,
1689                                        DMA_ATTR_WEAK_ORDERING);
1690
1691                 if (!skb) {
1692                         skb_free_frag(data);
1693                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1694                         cpr->sw_stats.rx.rx_oom_discards += 1;
1695                         return NULL;
1696                 }
1697                 skb_reserve(skb, bp->rx_offset);
1698                 skb_put(skb, len);
1699         }
1700
1701         if (agg_bufs) {
1702                 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1703                 if (!skb) {
1704                         /* Page reuse already handled by bnxt_rx_pages(). */
1705                         cpr->sw_stats.rx.rx_oom_discards += 1;
1706                         return NULL;
1707                 }
1708         }
1709
1710         skb->protocol =
1711                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1712
1713         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1714                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1715
1716         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1717             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1718                 __be16 vlan_proto = htons(tpa_info->metadata >>
1719                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1720                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1721
1722                 if (eth_type_vlan(vlan_proto)) {
1723                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1724                 } else {
1725                         dev_kfree_skb(skb);
1726                         return NULL;
1727                 }
1728         }
1729
1730         skb_checksum_none_assert(skb);
1731         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1732                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1733                 skb->csum_level =
1734                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1735         }
1736
1737         if (gro)
1738                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1739
1740         return skb;
1741 }
1742
1743 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1744                          struct rx_agg_cmp *rx_agg)
1745 {
1746         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1747         struct bnxt_tpa_info *tpa_info;
1748
1749         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1750         tpa_info = &rxr->rx_tpa[agg_id];
1751         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1752         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1753 }
1754
1755 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1756                              struct sk_buff *skb)
1757 {
1758         if (skb->dev != bp->dev) {
1759                 /* this packet belongs to a vf-rep */
1760                 bnxt_vf_rep_rx(bp, skb);
1761                 return;
1762         }
1763         skb_record_rx_queue(skb, bnapi->index);
1764         napi_gro_receive(&bnapi->napi, skb);
1765 }
1766
1767 /* returns the following:
1768  * 1       - 1 packet successfully received
1769  * 0       - successful TPA_START, packet not completed yet
1770  * -EBUSY  - completion ring does not have all the agg buffers yet
1771  * -ENOMEM - packet aborted due to out of memory
1772  * -EIO    - packet aborted due to hw error indicated in BD
1773  */
1774 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1775                        u32 *raw_cons, u8 *event)
1776 {
1777         struct bnxt_napi *bnapi = cpr->bnapi;
1778         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1779         struct net_device *dev = bp->dev;
1780         struct rx_cmp *rxcmp;
1781         struct rx_cmp_ext *rxcmp1;
1782         u32 tmp_raw_cons = *raw_cons;
1783         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1784         struct bnxt_sw_rx_bd *rx_buf;
1785         unsigned int len;
1786         u8 *data_ptr, agg_bufs, cmp_type;
1787         bool xdp_active = false;
1788         dma_addr_t dma_addr;
1789         struct sk_buff *skb;
1790         struct xdp_buff xdp;
1791         u32 flags, misc;
1792         void *data;
1793         int rc = 0;
1794
1795         rxcmp = (struct rx_cmp *)
1796                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1797
1798         cmp_type = RX_CMP_TYPE(rxcmp);
1799
1800         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1801                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1802                 goto next_rx_no_prod_no_len;
1803         }
1804
1805         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1806         cp_cons = RING_CMP(tmp_raw_cons);
1807         rxcmp1 = (struct rx_cmp_ext *)
1808                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1809
1810         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1811                 return -EBUSY;
1812
1813         /* The valid test of the entry must be done first before
1814          * reading any further.
1815          */
1816         dma_rmb();
1817         prod = rxr->rx_prod;
1818
1819         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1820                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1821                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1822
1823                 *event |= BNXT_RX_EVENT;
1824                 goto next_rx_no_prod_no_len;
1825
1826         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1827                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1828                                    (struct rx_tpa_end_cmp *)rxcmp,
1829                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1830
1831                 if (IS_ERR(skb))
1832                         return -EBUSY;
1833
1834                 rc = -ENOMEM;
1835                 if (likely(skb)) {
1836                         bnxt_deliver_skb(bp, bnapi, skb);
1837                         rc = 1;
1838                 }
1839                 *event |= BNXT_RX_EVENT;
1840                 goto next_rx_no_prod_no_len;
1841         }
1842
1843         cons = rxcmp->rx_cmp_opaque;
1844         if (unlikely(cons != rxr->rx_next_cons)) {
1845                 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1846
1847                 /* 0xffff is forced error, don't print it */
1848                 if (rxr->rx_next_cons != 0xffff)
1849                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1850                                     cons, rxr->rx_next_cons);
1851                 bnxt_sched_reset(bp, rxr);
1852                 if (rc1)
1853                         return rc1;
1854                 goto next_rx_no_prod_no_len;
1855         }
1856         rx_buf = &rxr->rx_buf_ring[cons];
1857         data = rx_buf->data;
1858         data_ptr = rx_buf->data_ptr;
1859         prefetch(data_ptr);
1860
1861         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1862         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1863
1864         if (agg_bufs) {
1865                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1866                         return -EBUSY;
1867
1868                 cp_cons = NEXT_CMP(cp_cons);
1869                 *event |= BNXT_AGG_EVENT;
1870         }
1871         *event |= BNXT_RX_EVENT;
1872
1873         rx_buf->data = NULL;
1874         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1875                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1876
1877                 bnxt_reuse_rx_data(rxr, cons, data);
1878                 if (agg_bufs)
1879                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1880                                                false);
1881
1882                 rc = -EIO;
1883                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1884                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1885                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1886                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1887                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1888                                                  rx_err);
1889                                 bnxt_sched_reset(bp, rxr);
1890                         }
1891                 }
1892                 goto next_rx_no_len;
1893         }
1894
1895         flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1896         len = flags >> RX_CMP_LEN_SHIFT;
1897         dma_addr = rx_buf->mapping;
1898
1899         if (bnxt_xdp_attached(bp, rxr)) {
1900                 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
1901                 if (agg_bufs) {
1902                         u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
1903                                                              cp_cons, agg_bufs,
1904                                                              false);
1905                         if (!frag_len) {
1906                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1907                                 rc = -ENOMEM;
1908                                 goto next_rx;
1909                         }
1910                 }
1911                 xdp_active = true;
1912         }
1913
1914         if (xdp_active) {
1915                 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
1916                         rc = 1;
1917                         goto next_rx;
1918                 }
1919         }
1920
1921         if (len <= bp->rx_copy_thresh) {
1922                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1923                 bnxt_reuse_rx_data(rxr, cons, data);
1924                 if (!skb) {
1925                         if (agg_bufs) {
1926                                 if (!xdp_active)
1927                                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1928                                                                agg_bufs, false);
1929                                 else
1930                                         bnxt_xdp_buff_frags_free(rxr, &xdp);
1931                         }
1932                         cpr->sw_stats.rx.rx_oom_discards += 1;
1933                         rc = -ENOMEM;
1934                         goto next_rx;
1935                 }
1936         } else {
1937                 u32 payload;
1938
1939                 if (rx_buf->data_ptr == data_ptr)
1940                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1941                 else
1942                         payload = 0;
1943                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1944                                       payload | len);
1945                 if (!skb) {
1946                         cpr->sw_stats.rx.rx_oom_discards += 1;
1947                         rc = -ENOMEM;
1948                         goto next_rx;
1949                 }
1950         }
1951
1952         if (agg_bufs) {
1953                 if (!xdp_active) {
1954                         skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
1955                         if (!skb) {
1956                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1957                                 rc = -ENOMEM;
1958                                 goto next_rx;
1959                         }
1960                 } else {
1961                         skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
1962                         if (!skb) {
1963                                 /* we should be able to free the old skb here */
1964                                 bnxt_xdp_buff_frags_free(rxr, &xdp);
1965                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1966                                 rc = -ENOMEM;
1967                                 goto next_rx;
1968                         }
1969                 }
1970         }
1971
1972         if (RX_CMP_HASH_VALID(rxcmp)) {
1973                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1974                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1975
1976                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1977                 if (hash_type != 1 && hash_type != 3)
1978                         type = PKT_HASH_TYPE_L3;
1979                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1980         }
1981
1982         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1983         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1984
1985         if ((rxcmp1->rx_cmp_flags2 &
1986              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1987             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1988                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1989                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1990                 __be16 vlan_proto = htons(meta_data >>
1991                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1992
1993                 if (eth_type_vlan(vlan_proto)) {
1994                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1995                 } else {
1996                         dev_kfree_skb(skb);
1997                         goto next_rx;
1998                 }
1999         }
2000
2001         skb_checksum_none_assert(skb);
2002         if (RX_CMP_L4_CS_OK(rxcmp1)) {
2003                 if (dev->features & NETIF_F_RXCSUM) {
2004                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2005                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2006                 }
2007         } else {
2008                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2009                         if (dev->features & NETIF_F_RXCSUM)
2010                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2011                 }
2012         }
2013
2014         if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) ==
2015                      RX_CMP_FLAGS_ITYPE_PTP_W_TS) || bp->ptp_all_rx_tstamp) {
2016                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2017                         u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
2018                         u64 ns, ts;
2019
2020                         if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2021                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2022
2023                                 spin_lock_bh(&ptp->ptp_lock);
2024                                 ns = timecounter_cyc2time(&ptp->tc, ts);
2025                                 spin_unlock_bh(&ptp->ptp_lock);
2026                                 memset(skb_hwtstamps(skb), 0,
2027                                        sizeof(*skb_hwtstamps(skb)));
2028                                 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2029                         }
2030                 }
2031         }
2032         bnxt_deliver_skb(bp, bnapi, skb);
2033         rc = 1;
2034
2035 next_rx:
2036         cpr->rx_packets += 1;
2037         cpr->rx_bytes += len;
2038
2039 next_rx_no_len:
2040         rxr->rx_prod = NEXT_RX(prod);
2041         rxr->rx_next_cons = NEXT_RX(cons);
2042
2043 next_rx_no_prod_no_len:
2044         *raw_cons = tmp_raw_cons;
2045
2046         return rc;
2047 }
2048
2049 /* In netpoll mode, if we are using a combined completion ring, we need to
2050  * discard the rx packets and recycle the buffers.
2051  */
2052 static int bnxt_force_rx_discard(struct bnxt *bp,
2053                                  struct bnxt_cp_ring_info *cpr,
2054                                  u32 *raw_cons, u8 *event)
2055 {
2056         u32 tmp_raw_cons = *raw_cons;
2057         struct rx_cmp_ext *rxcmp1;
2058         struct rx_cmp *rxcmp;
2059         u16 cp_cons;
2060         u8 cmp_type;
2061         int rc;
2062
2063         cp_cons = RING_CMP(tmp_raw_cons);
2064         rxcmp = (struct rx_cmp *)
2065                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2066
2067         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2068         cp_cons = RING_CMP(tmp_raw_cons);
2069         rxcmp1 = (struct rx_cmp_ext *)
2070                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2071
2072         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2073                 return -EBUSY;
2074
2075         /* The valid test of the entry must be done first before
2076          * reading any further.
2077          */
2078         dma_rmb();
2079         cmp_type = RX_CMP_TYPE(rxcmp);
2080         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2081                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2082                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2083         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2084                 struct rx_tpa_end_cmp_ext *tpa_end1;
2085
2086                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2087                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2088                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2089         }
2090         rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2091         if (rc && rc != -EBUSY)
2092                 cpr->sw_stats.rx.rx_netpoll_discards += 1;
2093         return rc;
2094 }
2095
2096 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2097 {
2098         struct bnxt_fw_health *fw_health = bp->fw_health;
2099         u32 reg = fw_health->regs[reg_idx];
2100         u32 reg_type, reg_off, val = 0;
2101
2102         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2103         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2104         switch (reg_type) {
2105         case BNXT_FW_HEALTH_REG_TYPE_CFG:
2106                 pci_read_config_dword(bp->pdev, reg_off, &val);
2107                 break;
2108         case BNXT_FW_HEALTH_REG_TYPE_GRC:
2109                 reg_off = fw_health->mapped_regs[reg_idx];
2110                 fallthrough;
2111         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2112                 val = readl(bp->bar0 + reg_off);
2113                 break;
2114         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2115                 val = readl(bp->bar1 + reg_off);
2116                 break;
2117         }
2118         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2119                 val &= fw_health->fw_reset_inprog_reg_mask;
2120         return val;
2121 }
2122
2123 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2124 {
2125         int i;
2126
2127         for (i = 0; i < bp->rx_nr_rings; i++) {
2128                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2129                 struct bnxt_ring_grp_info *grp_info;
2130
2131                 grp_info = &bp->grp_info[grp_idx];
2132                 if (grp_info->agg_fw_ring_id == ring_id)
2133                         return grp_idx;
2134         }
2135         return INVALID_HW_RING_ID;
2136 }
2137
2138 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2139 {
2140         u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2141
2142         switch (err_type) {
2143         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2144                 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2145                            BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2146                 break;
2147         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2148                 netdev_warn(bp->dev, "Pause Storm detected!\n");
2149                 break;
2150         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2151                 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2152                 break;
2153         default:
2154                 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2155                            err_type);
2156                 break;
2157         }
2158 }
2159
2160 #define BNXT_GET_EVENT_PORT(data)       \
2161         ((data) &                       \
2162          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2163
2164 #define BNXT_EVENT_RING_TYPE(data2)     \
2165         ((data2) &                      \
2166          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2167
2168 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
2169         (BNXT_EVENT_RING_TYPE(data2) == \
2170          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2171
2172 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)        \
2173         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2174          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2175
2176 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)        \
2177         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2178          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2179
2180 #define BNXT_PHC_BITS   48
2181
2182 static int bnxt_async_event_process(struct bnxt *bp,
2183                                     struct hwrm_async_event_cmpl *cmpl)
2184 {
2185         u16 event_id = le16_to_cpu(cmpl->event_id);
2186         u32 data1 = le32_to_cpu(cmpl->event_data1);
2187         u32 data2 = le32_to_cpu(cmpl->event_data2);
2188
2189         netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2190                    event_id, data1, data2);
2191
2192         /* TODO CHIMP_FW: Define event id's for link change, error etc */
2193         switch (event_id) {
2194         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2195                 struct bnxt_link_info *link_info = &bp->link_info;
2196
2197                 if (BNXT_VF(bp))
2198                         goto async_event_process_exit;
2199
2200                 /* print unsupported speed warning in forced speed mode only */
2201                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2202                     (data1 & 0x20000)) {
2203                         u16 fw_speed = link_info->force_link_speed;
2204                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2205
2206                         if (speed != SPEED_UNKNOWN)
2207                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2208                                             speed);
2209                 }
2210                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2211         }
2212                 fallthrough;
2213         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2214         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2215                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2216                 fallthrough;
2217         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2218                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2219                 break;
2220         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2221                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2222                 break;
2223         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2224                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2225
2226                 if (BNXT_VF(bp))
2227                         break;
2228
2229                 if (bp->pf.port_id != port_id)
2230                         break;
2231
2232                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2233                 break;
2234         }
2235         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2236                 if (BNXT_PF(bp))
2237                         goto async_event_process_exit;
2238                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2239                 break;
2240         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2241                 char *type_str = "Solicited";
2242
2243                 if (!bp->fw_health)
2244                         goto async_event_process_exit;
2245
2246                 bp->fw_reset_timestamp = jiffies;
2247                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2248                 if (!bp->fw_reset_min_dsecs)
2249                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2250                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2251                 if (!bp->fw_reset_max_dsecs)
2252                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2253                 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2254                         set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2255                 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2256                         type_str = "Fatal";
2257                         bp->fw_health->fatalities++;
2258                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2259                 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2260                            EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2261                         type_str = "Non-fatal";
2262                         bp->fw_health->survivals++;
2263                         set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2264                 }
2265                 netif_warn(bp, hw, bp->dev,
2266                            "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2267                            type_str, data1, data2,
2268                            bp->fw_reset_min_dsecs * 100,
2269                            bp->fw_reset_max_dsecs * 100);
2270                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2271                 break;
2272         }
2273         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2274                 struct bnxt_fw_health *fw_health = bp->fw_health;
2275                 char *status_desc = "healthy";
2276                 u32 status;
2277
2278                 if (!fw_health)
2279                         goto async_event_process_exit;
2280
2281                 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2282                         fw_health->enabled = false;
2283                         netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2284                         break;
2285                 }
2286                 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2287                 fw_health->tmr_multiplier =
2288                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2289                                      bp->current_interval * 10);
2290                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2291                 if (!fw_health->enabled)
2292                         fw_health->last_fw_heartbeat =
2293                                 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2294                 fw_health->last_fw_reset_cnt =
2295                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2296                 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2297                 if (status != BNXT_FW_STATUS_HEALTHY)
2298                         status_desc = "unhealthy";
2299                 netif_info(bp, drv, bp->dev,
2300                            "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2301                            fw_health->primary ? "primary" : "backup", status,
2302                            status_desc, fw_health->last_fw_reset_cnt);
2303                 if (!fw_health->enabled) {
2304                         /* Make sure tmr_counter is set and visible to
2305                          * bnxt_health_check() before setting enabled to true.
2306                          */
2307                         smp_wmb();
2308                         fw_health->enabled = true;
2309                 }
2310                 goto async_event_process_exit;
2311         }
2312         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2313                 netif_notice(bp, hw, bp->dev,
2314                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2315                              data1, data2);
2316                 goto async_event_process_exit;
2317         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2318                 struct bnxt_rx_ring_info *rxr;
2319                 u16 grp_idx;
2320
2321                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2322                         goto async_event_process_exit;
2323
2324                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2325                             BNXT_EVENT_RING_TYPE(data2), data1);
2326                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2327                         goto async_event_process_exit;
2328
2329                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2330                 if (grp_idx == INVALID_HW_RING_ID) {
2331                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2332                                     data1);
2333                         goto async_event_process_exit;
2334                 }
2335                 rxr = bp->bnapi[grp_idx]->rx_ring;
2336                 bnxt_sched_reset(bp, rxr);
2337                 goto async_event_process_exit;
2338         }
2339         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2340                 struct bnxt_fw_health *fw_health = bp->fw_health;
2341
2342                 netif_notice(bp, hw, bp->dev,
2343                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2344                              data1, data2);
2345                 if (fw_health) {
2346                         fw_health->echo_req_data1 = data1;
2347                         fw_health->echo_req_data2 = data2;
2348                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2349                         break;
2350                 }
2351                 goto async_event_process_exit;
2352         }
2353         case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2354                 bnxt_ptp_pps_event(bp, data1, data2);
2355                 goto async_event_process_exit;
2356         }
2357         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2358                 bnxt_event_error_report(bp, data1, data2);
2359                 goto async_event_process_exit;
2360         }
2361         case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2362                 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2363                 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2364                         if (BNXT_PTP_USE_RTC(bp)) {
2365                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2366                                 u64 ns;
2367
2368                                 spin_lock_bh(&ptp->ptp_lock);
2369                                 bnxt_ptp_update_current_time(bp);
2370                                 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2371                                        BNXT_PHC_BITS) | ptp->current_time);
2372                                 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2373                                 spin_unlock_bh(&ptp->ptp_lock);
2374                         }
2375                         break;
2376                 }
2377                 goto async_event_process_exit;
2378         }
2379         case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2380                 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2381
2382                 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2383                 goto async_event_process_exit;
2384         }
2385         default:
2386                 goto async_event_process_exit;
2387         }
2388         bnxt_queue_sp_work(bp);
2389 async_event_process_exit:
2390         return 0;
2391 }
2392
2393 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2394 {
2395         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2396         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2397         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2398                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2399
2400         switch (cmpl_type) {
2401         case CMPL_BASE_TYPE_HWRM_DONE:
2402                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2403                 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2404                 break;
2405
2406         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2407                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2408
2409                 if ((vf_id < bp->pf.first_vf_id) ||
2410                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2411                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2412                                    vf_id);
2413                         return -EINVAL;
2414                 }
2415
2416                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2417                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2418                 bnxt_queue_sp_work(bp);
2419                 break;
2420
2421         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2422                 bnxt_async_event_process(bp,
2423                                          (struct hwrm_async_event_cmpl *)txcmp);
2424                 break;
2425
2426         default:
2427                 break;
2428         }
2429
2430         return 0;
2431 }
2432
2433 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2434 {
2435         struct bnxt_napi *bnapi = dev_instance;
2436         struct bnxt *bp = bnapi->bp;
2437         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2438         u32 cons = RING_CMP(cpr->cp_raw_cons);
2439
2440         cpr->event_ctr++;
2441         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2442         napi_schedule(&bnapi->napi);
2443         return IRQ_HANDLED;
2444 }
2445
2446 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2447 {
2448         u32 raw_cons = cpr->cp_raw_cons;
2449         u16 cons = RING_CMP(raw_cons);
2450         struct tx_cmp *txcmp;
2451
2452         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2453
2454         return TX_CMP_VALID(txcmp, raw_cons);
2455 }
2456
2457 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2458 {
2459         struct bnxt_napi *bnapi = dev_instance;
2460         struct bnxt *bp = bnapi->bp;
2461         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2462         u32 cons = RING_CMP(cpr->cp_raw_cons);
2463         u32 int_status;
2464
2465         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2466
2467         if (!bnxt_has_work(bp, cpr)) {
2468                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2469                 /* return if erroneous interrupt */
2470                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2471                         return IRQ_NONE;
2472         }
2473
2474         /* disable ring IRQ */
2475         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2476
2477         /* Return here if interrupt is shared and is disabled. */
2478         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2479                 return IRQ_HANDLED;
2480
2481         napi_schedule(&bnapi->napi);
2482         return IRQ_HANDLED;
2483 }
2484
2485 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2486                             int budget)
2487 {
2488         struct bnxt_napi *bnapi = cpr->bnapi;
2489         u32 raw_cons = cpr->cp_raw_cons;
2490         u32 cons;
2491         int tx_pkts = 0;
2492         int rx_pkts = 0;
2493         u8 event = 0;
2494         struct tx_cmp *txcmp;
2495
2496         cpr->has_more_work = 0;
2497         cpr->had_work_done = 1;
2498         while (1) {
2499                 int rc;
2500
2501                 cons = RING_CMP(raw_cons);
2502                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2503
2504                 if (!TX_CMP_VALID(txcmp, raw_cons))
2505                         break;
2506
2507                 /* The valid test of the entry must be done first before
2508                  * reading any further.
2509                  */
2510                 dma_rmb();
2511                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2512                         tx_pkts++;
2513                         /* return full budget so NAPI will complete. */
2514                         if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
2515                                 rx_pkts = budget;
2516                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2517                                 if (budget)
2518                                         cpr->has_more_work = 1;
2519                                 break;
2520                         }
2521                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2522                         if (likely(budget))
2523                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2524                         else
2525                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2526                                                            &event);
2527                         if (likely(rc >= 0))
2528                                 rx_pkts += rc;
2529                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2530                          * the NAPI budget.  Otherwise, we may potentially loop
2531                          * here forever if we consistently cannot allocate
2532                          * buffers.
2533                          */
2534                         else if (rc == -ENOMEM && budget)
2535                                 rx_pkts++;
2536                         else if (rc == -EBUSY)  /* partial completion */
2537                                 break;
2538                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2539                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2540                                     (TX_CMP_TYPE(txcmp) ==
2541                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2542                                     (TX_CMP_TYPE(txcmp) ==
2543                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2544                         bnxt_hwrm_handler(bp, txcmp);
2545                 }
2546                 raw_cons = NEXT_RAW_CMP(raw_cons);
2547
2548                 if (rx_pkts && rx_pkts == budget) {
2549                         cpr->has_more_work = 1;
2550                         break;
2551                 }
2552         }
2553
2554         if (event & BNXT_REDIRECT_EVENT)
2555                 xdp_do_flush();
2556
2557         if (event & BNXT_TX_EVENT) {
2558                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2559                 u16 prod = txr->tx_prod;
2560
2561                 /* Sync BD data before updating doorbell */
2562                 wmb();
2563
2564                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2565         }
2566
2567         cpr->cp_raw_cons = raw_cons;
2568         bnapi->tx_pkts += tx_pkts;
2569         bnapi->events |= event;
2570         return rx_pkts;
2571 }
2572
2573 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2574 {
2575         if (bnapi->tx_pkts) {
2576                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2577                 bnapi->tx_pkts = 0;
2578         }
2579
2580         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2581                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2582
2583                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2584         }
2585         if (bnapi->events & BNXT_AGG_EVENT) {
2586                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2587
2588                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2589         }
2590         bnapi->events = 0;
2591 }
2592
2593 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2594                           int budget)
2595 {
2596         struct bnxt_napi *bnapi = cpr->bnapi;
2597         int rx_pkts;
2598
2599         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2600
2601         /* ACK completion ring before freeing tx ring and producing new
2602          * buffers in rx/agg rings to prevent overflowing the completion
2603          * ring.
2604          */
2605         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2606
2607         __bnxt_poll_work_done(bp, bnapi);
2608         return rx_pkts;
2609 }
2610
2611 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2612 {
2613         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2614         struct bnxt *bp = bnapi->bp;
2615         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2616         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2617         struct tx_cmp *txcmp;
2618         struct rx_cmp_ext *rxcmp1;
2619         u32 cp_cons, tmp_raw_cons;
2620         u32 raw_cons = cpr->cp_raw_cons;
2621         u32 rx_pkts = 0;
2622         u8 event = 0;
2623
2624         while (1) {
2625                 int rc;
2626
2627                 cp_cons = RING_CMP(raw_cons);
2628                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2629
2630                 if (!TX_CMP_VALID(txcmp, raw_cons))
2631                         break;
2632
2633                 /* The valid test of the entry must be done first before
2634                  * reading any further.
2635                  */
2636                 dma_rmb();
2637                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2638                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2639                         cp_cons = RING_CMP(tmp_raw_cons);
2640                         rxcmp1 = (struct rx_cmp_ext *)
2641                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2642
2643                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2644                                 break;
2645
2646                         /* force an error to recycle the buffer */
2647                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2648                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2649
2650                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2651                         if (likely(rc == -EIO) && budget)
2652                                 rx_pkts++;
2653                         else if (rc == -EBUSY)  /* partial completion */
2654                                 break;
2655                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2656                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2657                         bnxt_hwrm_handler(bp, txcmp);
2658                 } else {
2659                         netdev_err(bp->dev,
2660                                    "Invalid completion received on special ring\n");
2661                 }
2662                 raw_cons = NEXT_RAW_CMP(raw_cons);
2663
2664                 if (rx_pkts == budget)
2665                         break;
2666         }
2667
2668         cpr->cp_raw_cons = raw_cons;
2669         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2670         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2671
2672         if (event & BNXT_AGG_EVENT)
2673                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2674
2675         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2676                 napi_complete_done(napi, rx_pkts);
2677                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2678         }
2679         return rx_pkts;
2680 }
2681
2682 static int bnxt_poll(struct napi_struct *napi, int budget)
2683 {
2684         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2685         struct bnxt *bp = bnapi->bp;
2686         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2687         int work_done = 0;
2688
2689         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2690                 napi_complete(napi);
2691                 return 0;
2692         }
2693         while (1) {
2694                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2695
2696                 if (work_done >= budget) {
2697                         if (!budget)
2698                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2699                         break;
2700                 }
2701
2702                 if (!bnxt_has_work(bp, cpr)) {
2703                         if (napi_complete_done(napi, work_done))
2704                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2705                         break;
2706                 }
2707         }
2708         if (bp->flags & BNXT_FLAG_DIM) {
2709                 struct dim_sample dim_sample = {};
2710
2711                 dim_update_sample(cpr->event_ctr,
2712                                   cpr->rx_packets,
2713                                   cpr->rx_bytes,
2714                                   &dim_sample);
2715                 net_dim(&cpr->dim, dim_sample);
2716         }
2717         return work_done;
2718 }
2719
2720 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2721 {
2722         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2723         int i, work_done = 0;
2724
2725         for (i = 0; i < 2; i++) {
2726                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2727
2728                 if (cpr2) {
2729                         work_done += __bnxt_poll_work(bp, cpr2,
2730                                                       budget - work_done);
2731                         cpr->has_more_work |= cpr2->has_more_work;
2732                 }
2733         }
2734         return work_done;
2735 }
2736
2737 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2738                                  u64 dbr_type)
2739 {
2740         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2741         int i;
2742
2743         for (i = 0; i < 2; i++) {
2744                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2745                 struct bnxt_db_info *db;
2746
2747                 if (cpr2 && cpr2->had_work_done) {
2748                         db = &cpr2->cp_db;
2749                         bnxt_writeq(bp, db->db_key64 | dbr_type |
2750                                     RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2751                         cpr2->had_work_done = 0;
2752                 }
2753         }
2754         __bnxt_poll_work_done(bp, bnapi);
2755 }
2756
2757 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2758 {
2759         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2760         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2761         struct bnxt_cp_ring_info *cpr_rx;
2762         u32 raw_cons = cpr->cp_raw_cons;
2763         struct bnxt *bp = bnapi->bp;
2764         struct nqe_cn *nqcmp;
2765         int work_done = 0;
2766         u32 cons;
2767
2768         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2769                 napi_complete(napi);
2770                 return 0;
2771         }
2772         if (cpr->has_more_work) {
2773                 cpr->has_more_work = 0;
2774                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2775         }
2776         while (1) {
2777                 cons = RING_CMP(raw_cons);
2778                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2779
2780                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2781                         if (cpr->has_more_work)
2782                                 break;
2783
2784                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2785                         cpr->cp_raw_cons = raw_cons;
2786                         if (napi_complete_done(napi, work_done))
2787                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2788                                                   cpr->cp_raw_cons);
2789                         goto poll_done;
2790                 }
2791
2792                 /* The valid test of the entry must be done first before
2793                  * reading any further.
2794                  */
2795                 dma_rmb();
2796
2797                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2798                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2799                         struct bnxt_cp_ring_info *cpr2;
2800
2801                         /* No more budget for RX work */
2802                         if (budget && work_done >= budget && idx == BNXT_RX_HDL)
2803                                 break;
2804
2805                         cpr2 = cpr->cp_ring_arr[idx];
2806                         work_done += __bnxt_poll_work(bp, cpr2,
2807                                                       budget - work_done);
2808                         cpr->has_more_work |= cpr2->has_more_work;
2809                 } else {
2810                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2811                 }
2812                 raw_cons = NEXT_RAW_CMP(raw_cons);
2813         }
2814         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2815         if (raw_cons != cpr->cp_raw_cons) {
2816                 cpr->cp_raw_cons = raw_cons;
2817                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2818         }
2819 poll_done:
2820         cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
2821         if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
2822                 struct dim_sample dim_sample = {};
2823
2824                 dim_update_sample(cpr->event_ctr,
2825                                   cpr_rx->rx_packets,
2826                                   cpr_rx->rx_bytes,
2827                                   &dim_sample);
2828                 net_dim(&cpr->dim, dim_sample);
2829         }
2830         return work_done;
2831 }
2832
2833 static void bnxt_free_tx_skbs(struct bnxt *bp)
2834 {
2835         int i, max_idx;
2836         struct pci_dev *pdev = bp->pdev;
2837
2838         if (!bp->tx_ring)
2839                 return;
2840
2841         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2842         for (i = 0; i < bp->tx_nr_rings; i++) {
2843                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2844                 int j;
2845
2846                 if (!txr->tx_buf_ring)
2847                         continue;
2848
2849                 for (j = 0; j < max_idx;) {
2850                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2851                         struct sk_buff *skb;
2852                         int k, last;
2853
2854                         if (i < bp->tx_nr_rings_xdp &&
2855                             tx_buf->action == XDP_REDIRECT) {
2856                                 dma_unmap_single(&pdev->dev,
2857                                         dma_unmap_addr(tx_buf, mapping),
2858                                         dma_unmap_len(tx_buf, len),
2859                                         DMA_TO_DEVICE);
2860                                 xdp_return_frame(tx_buf->xdpf);
2861                                 tx_buf->action = 0;
2862                                 tx_buf->xdpf = NULL;
2863                                 j++;
2864                                 continue;
2865                         }
2866
2867                         skb = tx_buf->skb;
2868                         if (!skb) {
2869                                 j++;
2870                                 continue;
2871                         }
2872
2873                         tx_buf->skb = NULL;
2874
2875                         if (tx_buf->is_push) {
2876                                 dev_kfree_skb(skb);
2877                                 j += 2;
2878                                 continue;
2879                         }
2880
2881                         dma_unmap_single(&pdev->dev,
2882                                          dma_unmap_addr(tx_buf, mapping),
2883                                          skb_headlen(skb),
2884                                          DMA_TO_DEVICE);
2885
2886                         last = tx_buf->nr_frags;
2887                         j += 2;
2888                         for (k = 0; k < last; k++, j++) {
2889                                 int ring_idx = j & bp->tx_ring_mask;
2890                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2891
2892                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2893                                 dma_unmap_page(
2894                                         &pdev->dev,
2895                                         dma_unmap_addr(tx_buf, mapping),
2896                                         skb_frag_size(frag), DMA_TO_DEVICE);
2897                         }
2898                         dev_kfree_skb(skb);
2899                 }
2900                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2901         }
2902 }
2903
2904 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2905 {
2906         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2907         struct pci_dev *pdev = bp->pdev;
2908         struct bnxt_tpa_idx_map *map;
2909         int i, max_idx, max_agg_idx;
2910
2911         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2912         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2913         if (!rxr->rx_tpa)
2914                 goto skip_rx_tpa_free;
2915
2916         for (i = 0; i < bp->max_tpa; i++) {
2917                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2918                 u8 *data = tpa_info->data;
2919
2920                 if (!data)
2921                         continue;
2922
2923                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2924                                        bp->rx_buf_use_size, bp->rx_dir,
2925                                        DMA_ATTR_WEAK_ORDERING);
2926
2927                 tpa_info->data = NULL;
2928
2929                 skb_free_frag(data);
2930         }
2931
2932 skip_rx_tpa_free:
2933         if (!rxr->rx_buf_ring)
2934                 goto skip_rx_buf_free;
2935
2936         for (i = 0; i < max_idx; i++) {
2937                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2938                 dma_addr_t mapping = rx_buf->mapping;
2939                 void *data = rx_buf->data;
2940
2941                 if (!data)
2942                         continue;
2943
2944                 rx_buf->data = NULL;
2945                 if (BNXT_RX_PAGE_MODE(bp)) {
2946                         mapping -= bp->rx_dma_offset;
2947                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2948                                              bp->rx_dir,
2949                                              DMA_ATTR_WEAK_ORDERING);
2950                         page_pool_recycle_direct(rxr->page_pool, data);
2951                 } else {
2952                         dma_unmap_single_attrs(&pdev->dev, mapping,
2953                                                bp->rx_buf_use_size, bp->rx_dir,
2954                                                DMA_ATTR_WEAK_ORDERING);
2955                         skb_free_frag(data);
2956                 }
2957         }
2958
2959 skip_rx_buf_free:
2960         if (!rxr->rx_agg_ring)
2961                 goto skip_rx_agg_free;
2962
2963         for (i = 0; i < max_agg_idx; i++) {
2964                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2965                 struct page *page = rx_agg_buf->page;
2966
2967                 if (!page)
2968                         continue;
2969
2970                 if (BNXT_RX_PAGE_MODE(bp)) {
2971                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2972                                              BNXT_RX_PAGE_SIZE, bp->rx_dir,
2973                                              DMA_ATTR_WEAK_ORDERING);
2974                         rx_agg_buf->page = NULL;
2975                         __clear_bit(i, rxr->rx_agg_bmap);
2976
2977                         page_pool_recycle_direct(rxr->page_pool, page);
2978                 } else {
2979                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2980                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
2981                                              DMA_ATTR_WEAK_ORDERING);
2982                         rx_agg_buf->page = NULL;
2983                         __clear_bit(i, rxr->rx_agg_bmap);
2984
2985                         __free_page(page);
2986                 }
2987         }
2988
2989 skip_rx_agg_free:
2990         if (rxr->rx_page) {
2991                 __free_page(rxr->rx_page);
2992                 rxr->rx_page = NULL;
2993         }
2994         map = rxr->rx_tpa_idx_map;
2995         if (map)
2996                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2997 }
2998
2999 static void bnxt_free_rx_skbs(struct bnxt *bp)
3000 {
3001         int i;
3002
3003         if (!bp->rx_ring)
3004                 return;
3005
3006         for (i = 0; i < bp->rx_nr_rings; i++)
3007                 bnxt_free_one_rx_ring_skbs(bp, i);
3008 }
3009
3010 static void bnxt_free_skbs(struct bnxt *bp)
3011 {
3012         bnxt_free_tx_skbs(bp);
3013         bnxt_free_rx_skbs(bp);
3014 }
3015
3016 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
3017 {
3018         u8 init_val = mem_init->init_val;
3019         u16 offset = mem_init->offset;
3020         u8 *p2 = p;
3021         int i;
3022
3023         if (!init_val)
3024                 return;
3025         if (offset == BNXT_MEM_INVALID_OFFSET) {
3026                 memset(p, init_val, len);
3027                 return;
3028         }
3029         for (i = 0; i < len; i += mem_init->size)
3030                 *(p2 + i + offset) = init_val;
3031 }
3032
3033 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3034 {
3035         struct pci_dev *pdev = bp->pdev;
3036         int i;
3037
3038         if (!rmem->pg_arr)
3039                 goto skip_pages;
3040
3041         for (i = 0; i < rmem->nr_pages; i++) {
3042                 if (!rmem->pg_arr[i])
3043                         continue;
3044
3045                 dma_free_coherent(&pdev->dev, rmem->page_size,
3046                                   rmem->pg_arr[i], rmem->dma_arr[i]);
3047
3048                 rmem->pg_arr[i] = NULL;
3049         }
3050 skip_pages:
3051         if (rmem->pg_tbl) {
3052                 size_t pg_tbl_size = rmem->nr_pages * 8;
3053
3054                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3055                         pg_tbl_size = rmem->page_size;
3056                 dma_free_coherent(&pdev->dev, pg_tbl_size,
3057                                   rmem->pg_tbl, rmem->pg_tbl_map);
3058                 rmem->pg_tbl = NULL;
3059         }
3060         if (rmem->vmem_size && *rmem->vmem) {
3061                 vfree(*rmem->vmem);
3062                 *rmem->vmem = NULL;
3063         }
3064 }
3065
3066 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3067 {
3068         struct pci_dev *pdev = bp->pdev;
3069         u64 valid_bit = 0;
3070         int i;
3071
3072         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3073                 valid_bit = PTU_PTE_VALID;
3074         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3075                 size_t pg_tbl_size = rmem->nr_pages * 8;
3076
3077                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3078                         pg_tbl_size = rmem->page_size;
3079                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3080                                                   &rmem->pg_tbl_map,
3081                                                   GFP_KERNEL);
3082                 if (!rmem->pg_tbl)
3083                         return -ENOMEM;
3084         }
3085
3086         for (i = 0; i < rmem->nr_pages; i++) {
3087                 u64 extra_bits = valid_bit;
3088
3089                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3090                                                      rmem->page_size,
3091                                                      &rmem->dma_arr[i],
3092                                                      GFP_KERNEL);
3093                 if (!rmem->pg_arr[i])
3094                         return -ENOMEM;
3095
3096                 if (rmem->mem_init)
3097                         bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
3098                                           rmem->page_size);
3099                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3100                         if (i == rmem->nr_pages - 2 &&
3101                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3102                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3103                         else if (i == rmem->nr_pages - 1 &&
3104                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3105                                 extra_bits |= PTU_PTE_LAST;
3106                         rmem->pg_tbl[i] =
3107                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3108                 }
3109         }
3110
3111         if (rmem->vmem_size) {
3112                 *rmem->vmem = vzalloc(rmem->vmem_size);
3113                 if (!(*rmem->vmem))
3114                         return -ENOMEM;
3115         }
3116         return 0;
3117 }
3118
3119 static void bnxt_free_tpa_info(struct bnxt *bp)
3120 {
3121         int i, j;
3122
3123         for (i = 0; i < bp->rx_nr_rings; i++) {
3124                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3125
3126                 kfree(rxr->rx_tpa_idx_map);
3127                 rxr->rx_tpa_idx_map = NULL;
3128                 if (rxr->rx_tpa) {
3129                         for (j = 0; j < bp->max_tpa; j++) {
3130                                 kfree(rxr->rx_tpa[j].agg_arr);
3131                                 rxr->rx_tpa[j].agg_arr = NULL;
3132                         }
3133                 }
3134                 kfree(rxr->rx_tpa);
3135                 rxr->rx_tpa = NULL;
3136         }
3137 }
3138
3139 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3140 {
3141         int i, j;
3142
3143         bp->max_tpa = MAX_TPA;
3144         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3145                 if (!bp->max_tpa_v2)
3146                         return 0;
3147                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3148         }
3149
3150         for (i = 0; i < bp->rx_nr_rings; i++) {
3151                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3152                 struct rx_agg_cmp *agg;
3153
3154                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3155                                       GFP_KERNEL);
3156                 if (!rxr->rx_tpa)
3157                         return -ENOMEM;
3158
3159                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3160                         continue;
3161                 for (j = 0; j < bp->max_tpa; j++) {
3162                         agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3163                         if (!agg)
3164                                 return -ENOMEM;
3165                         rxr->rx_tpa[j].agg_arr = agg;
3166                 }
3167                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3168                                               GFP_KERNEL);
3169                 if (!rxr->rx_tpa_idx_map)
3170                         return -ENOMEM;
3171         }
3172         return 0;
3173 }
3174
3175 static void bnxt_free_rx_rings(struct bnxt *bp)
3176 {
3177         int i;
3178
3179         if (!bp->rx_ring)
3180                 return;
3181
3182         bnxt_free_tpa_info(bp);
3183         for (i = 0; i < bp->rx_nr_rings; i++) {
3184                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3185                 struct bnxt_ring_struct *ring;
3186
3187                 if (rxr->xdp_prog)
3188                         bpf_prog_put(rxr->xdp_prog);
3189
3190                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3191                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3192
3193                 page_pool_destroy(rxr->page_pool);
3194                 rxr->page_pool = NULL;
3195
3196                 kfree(rxr->rx_agg_bmap);
3197                 rxr->rx_agg_bmap = NULL;
3198
3199                 ring = &rxr->rx_ring_struct;
3200                 bnxt_free_ring(bp, &ring->ring_mem);
3201
3202                 ring = &rxr->rx_agg_ring_struct;
3203                 bnxt_free_ring(bp, &ring->ring_mem);
3204         }
3205 }
3206
3207 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3208                                    struct bnxt_rx_ring_info *rxr)
3209 {
3210         struct page_pool_params pp = { 0 };
3211
3212         pp.pool_size = bp->rx_ring_size;
3213         pp.nid = dev_to_node(&bp->pdev->dev);
3214         pp.napi = &rxr->bnapi->napi;
3215         pp.dev = &bp->pdev->dev;
3216         pp.dma_dir = DMA_BIDIRECTIONAL;
3217
3218         rxr->page_pool = page_pool_create(&pp);
3219         if (IS_ERR(rxr->page_pool)) {
3220                 int err = PTR_ERR(rxr->page_pool);
3221
3222                 rxr->page_pool = NULL;
3223                 return err;
3224         }
3225         return 0;
3226 }
3227
3228 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3229 {
3230         int i, rc = 0, agg_rings = 0;
3231
3232         if (!bp->rx_ring)
3233                 return -ENOMEM;
3234
3235         if (bp->flags & BNXT_FLAG_AGG_RINGS)
3236                 agg_rings = 1;
3237
3238         for (i = 0; i < bp->rx_nr_rings; i++) {
3239                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3240                 struct bnxt_ring_struct *ring;
3241
3242                 ring = &rxr->rx_ring_struct;
3243
3244                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3245                 if (rc)
3246                         return rc;
3247
3248                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3249                 if (rc < 0)
3250                         return rc;
3251
3252                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3253                                                 MEM_TYPE_PAGE_POOL,
3254                                                 rxr->page_pool);
3255                 if (rc) {
3256                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3257                         return rc;
3258                 }
3259
3260                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3261                 if (rc)
3262                         return rc;
3263
3264                 ring->grp_idx = i;
3265                 if (agg_rings) {
3266                         u16 mem_size;
3267
3268                         ring = &rxr->rx_agg_ring_struct;
3269                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3270                         if (rc)
3271                                 return rc;
3272
3273                         ring->grp_idx = i;
3274                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3275                         mem_size = rxr->rx_agg_bmap_size / 8;
3276                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3277                         if (!rxr->rx_agg_bmap)
3278                                 return -ENOMEM;
3279                 }
3280         }
3281         if (bp->flags & BNXT_FLAG_TPA)
3282                 rc = bnxt_alloc_tpa_info(bp);
3283         return rc;
3284 }
3285
3286 static void bnxt_free_tx_rings(struct bnxt *bp)
3287 {
3288         int i;
3289         struct pci_dev *pdev = bp->pdev;
3290
3291         if (!bp->tx_ring)
3292                 return;
3293
3294         for (i = 0; i < bp->tx_nr_rings; i++) {
3295                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3296                 struct bnxt_ring_struct *ring;
3297
3298                 if (txr->tx_push) {
3299                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
3300                                           txr->tx_push, txr->tx_push_mapping);
3301                         txr->tx_push = NULL;
3302                 }
3303
3304                 ring = &txr->tx_ring_struct;
3305
3306                 bnxt_free_ring(bp, &ring->ring_mem);
3307         }
3308 }
3309
3310 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3311 {
3312         int i, j, rc;
3313         struct pci_dev *pdev = bp->pdev;
3314
3315         bp->tx_push_size = 0;
3316         if (bp->tx_push_thresh) {
3317                 int push_size;
3318
3319                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3320                                         bp->tx_push_thresh);
3321
3322                 if (push_size > 256) {
3323                         push_size = 0;
3324                         bp->tx_push_thresh = 0;
3325                 }
3326
3327                 bp->tx_push_size = push_size;
3328         }
3329
3330         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3331                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3332                 struct bnxt_ring_struct *ring;
3333                 u8 qidx;
3334
3335                 ring = &txr->tx_ring_struct;
3336
3337                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3338                 if (rc)
3339                         return rc;
3340
3341                 ring->grp_idx = txr->bnapi->index;
3342                 if (bp->tx_push_size) {
3343                         dma_addr_t mapping;
3344
3345                         /* One pre-allocated DMA buffer to backup
3346                          * TX push operation
3347                          */
3348                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3349                                                 bp->tx_push_size,
3350                                                 &txr->tx_push_mapping,
3351                                                 GFP_KERNEL);
3352
3353                         if (!txr->tx_push)
3354                                 return -ENOMEM;
3355
3356                         mapping = txr->tx_push_mapping +
3357                                 sizeof(struct tx_push_bd);
3358                         txr->data_mapping = cpu_to_le64(mapping);
3359                 }
3360                 qidx = bp->tc_to_qidx[j];
3361                 ring->queue_id = bp->q_info[qidx].queue_id;
3362                 spin_lock_init(&txr->xdp_tx_lock);
3363                 if (i < bp->tx_nr_rings_xdp)
3364                         continue;
3365                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3366                         j++;
3367         }
3368         return 0;
3369 }
3370
3371 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3372 {
3373         struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3374
3375         kfree(cpr->cp_desc_ring);
3376         cpr->cp_desc_ring = NULL;
3377         ring->ring_mem.pg_arr = NULL;
3378         kfree(cpr->cp_desc_mapping);
3379         cpr->cp_desc_mapping = NULL;
3380         ring->ring_mem.dma_arr = NULL;
3381 }
3382
3383 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3384 {
3385         cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3386         if (!cpr->cp_desc_ring)
3387                 return -ENOMEM;
3388         cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3389                                        GFP_KERNEL);
3390         if (!cpr->cp_desc_mapping)
3391                 return -ENOMEM;
3392         return 0;
3393 }
3394
3395 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3396 {
3397         int i;
3398
3399         if (!bp->bnapi)
3400                 return;
3401         for (i = 0; i < bp->cp_nr_rings; i++) {
3402                 struct bnxt_napi *bnapi = bp->bnapi[i];
3403
3404                 if (!bnapi)
3405                         continue;
3406                 bnxt_free_cp_arrays(&bnapi->cp_ring);
3407         }
3408 }
3409
3410 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3411 {
3412         int i, n = bp->cp_nr_pages;
3413
3414         for (i = 0; i < bp->cp_nr_rings; i++) {
3415                 struct bnxt_napi *bnapi = bp->bnapi[i];
3416                 int rc;
3417
3418                 if (!bnapi)
3419                         continue;
3420                 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3421                 if (rc)
3422                         return rc;
3423         }
3424         return 0;
3425 }
3426
3427 static void bnxt_free_cp_rings(struct bnxt *bp)
3428 {
3429         int i;
3430
3431         if (!bp->bnapi)
3432                 return;
3433
3434         for (i = 0; i < bp->cp_nr_rings; i++) {
3435                 struct bnxt_napi *bnapi = bp->bnapi[i];
3436                 struct bnxt_cp_ring_info *cpr;
3437                 struct bnxt_ring_struct *ring;
3438                 int j;
3439
3440                 if (!bnapi)
3441                         continue;
3442
3443                 cpr = &bnapi->cp_ring;
3444                 ring = &cpr->cp_ring_struct;
3445
3446                 bnxt_free_ring(bp, &ring->ring_mem);
3447
3448                 for (j = 0; j < 2; j++) {
3449                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3450
3451                         if (cpr2) {
3452                                 ring = &cpr2->cp_ring_struct;
3453                                 bnxt_free_ring(bp, &ring->ring_mem);
3454                                 bnxt_free_cp_arrays(cpr2);
3455                                 kfree(cpr2);
3456                                 cpr->cp_ring_arr[j] = NULL;
3457                         }
3458                 }
3459         }
3460 }
3461
3462 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3463 {
3464         struct bnxt_ring_mem_info *rmem;
3465         struct bnxt_ring_struct *ring;
3466         struct bnxt_cp_ring_info *cpr;
3467         int rc;
3468
3469         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3470         if (!cpr)
3471                 return NULL;
3472
3473         rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3474         if (rc) {
3475                 bnxt_free_cp_arrays(cpr);
3476                 kfree(cpr);
3477                 return NULL;
3478         }
3479         ring = &cpr->cp_ring_struct;
3480         rmem = &ring->ring_mem;
3481         rmem->nr_pages = bp->cp_nr_pages;
3482         rmem->page_size = HW_CMPD_RING_SIZE;
3483         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3484         rmem->dma_arr = cpr->cp_desc_mapping;
3485         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3486         rc = bnxt_alloc_ring(bp, rmem);
3487         if (rc) {
3488                 bnxt_free_ring(bp, rmem);
3489                 bnxt_free_cp_arrays(cpr);
3490                 kfree(cpr);
3491                 cpr = NULL;
3492         }
3493         return cpr;
3494 }
3495
3496 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3497 {
3498         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3499         int i, rc, ulp_base_vec, ulp_msix;
3500
3501         ulp_msix = bnxt_get_ulp_msix_num(bp);
3502         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3503         for (i = 0; i < bp->cp_nr_rings; i++) {
3504                 struct bnxt_napi *bnapi = bp->bnapi[i];
3505                 struct bnxt_cp_ring_info *cpr;
3506                 struct bnxt_ring_struct *ring;
3507
3508                 if (!bnapi)
3509                         continue;
3510
3511                 cpr = &bnapi->cp_ring;
3512                 cpr->bnapi = bnapi;
3513                 ring = &cpr->cp_ring_struct;
3514
3515                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3516                 if (rc)
3517                         return rc;
3518
3519                 if (ulp_msix && i >= ulp_base_vec)
3520                         ring->map_idx = i + ulp_msix;
3521                 else
3522                         ring->map_idx = i;
3523
3524                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3525                         continue;
3526
3527                 if (i < bp->rx_nr_rings) {
3528                         struct bnxt_cp_ring_info *cpr2 =
3529                                 bnxt_alloc_cp_sub_ring(bp);
3530
3531                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3532                         if (!cpr2)
3533                                 return -ENOMEM;
3534                         cpr2->bnapi = bnapi;
3535                 }
3536                 if ((sh && i < bp->tx_nr_rings) ||
3537                     (!sh && i >= bp->rx_nr_rings)) {
3538                         struct bnxt_cp_ring_info *cpr2 =
3539                                 bnxt_alloc_cp_sub_ring(bp);
3540
3541                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3542                         if (!cpr2)
3543                                 return -ENOMEM;
3544                         cpr2->bnapi = bnapi;
3545                 }
3546         }
3547         return 0;
3548 }
3549
3550 static void bnxt_init_ring_struct(struct bnxt *bp)
3551 {
3552         int i;
3553
3554         for (i = 0; i < bp->cp_nr_rings; i++) {
3555                 struct bnxt_napi *bnapi = bp->bnapi[i];
3556                 struct bnxt_ring_mem_info *rmem;
3557                 struct bnxt_cp_ring_info *cpr;
3558                 struct bnxt_rx_ring_info *rxr;
3559                 struct bnxt_tx_ring_info *txr;
3560                 struct bnxt_ring_struct *ring;
3561
3562                 if (!bnapi)
3563                         continue;
3564
3565                 cpr = &bnapi->cp_ring;
3566                 ring = &cpr->cp_ring_struct;
3567                 rmem = &ring->ring_mem;
3568                 rmem->nr_pages = bp->cp_nr_pages;
3569                 rmem->page_size = HW_CMPD_RING_SIZE;
3570                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3571                 rmem->dma_arr = cpr->cp_desc_mapping;
3572                 rmem->vmem_size = 0;
3573
3574                 rxr = bnapi->rx_ring;
3575                 if (!rxr)
3576                         goto skip_rx;
3577
3578                 ring = &rxr->rx_ring_struct;
3579                 rmem = &ring->ring_mem;
3580                 rmem->nr_pages = bp->rx_nr_pages;
3581                 rmem->page_size = HW_RXBD_RING_SIZE;
3582                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3583                 rmem->dma_arr = rxr->rx_desc_mapping;
3584                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3585                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3586
3587                 ring = &rxr->rx_agg_ring_struct;
3588                 rmem = &ring->ring_mem;
3589                 rmem->nr_pages = bp->rx_agg_nr_pages;
3590                 rmem->page_size = HW_RXBD_RING_SIZE;
3591                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3592                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3593                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3594                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3595
3596 skip_rx:
3597                 txr = bnapi->tx_ring;
3598                 if (!txr)
3599                         continue;
3600
3601                 ring = &txr->tx_ring_struct;
3602                 rmem = &ring->ring_mem;
3603                 rmem->nr_pages = bp->tx_nr_pages;
3604                 rmem->page_size = HW_RXBD_RING_SIZE;
3605                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3606                 rmem->dma_arr = txr->tx_desc_mapping;
3607                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3608                 rmem->vmem = (void **)&txr->tx_buf_ring;
3609         }
3610 }
3611
3612 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3613 {
3614         int i;
3615         u32 prod;
3616         struct rx_bd **rx_buf_ring;
3617
3618         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3619         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3620                 int j;
3621                 struct rx_bd *rxbd;
3622
3623                 rxbd = rx_buf_ring[i];
3624                 if (!rxbd)
3625                         continue;
3626
3627                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3628                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3629                         rxbd->rx_bd_opaque = prod;
3630                 }
3631         }
3632 }
3633
3634 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3635 {
3636         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3637         struct net_device *dev = bp->dev;
3638         u32 prod;
3639         int i;
3640
3641         prod = rxr->rx_prod;
3642         for (i = 0; i < bp->rx_ring_size; i++) {
3643                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3644                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3645                                     ring_nr, i, bp->rx_ring_size);
3646                         break;
3647                 }
3648                 prod = NEXT_RX(prod);
3649         }
3650         rxr->rx_prod = prod;
3651
3652         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3653                 return 0;
3654
3655         prod = rxr->rx_agg_prod;
3656         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3657                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3658                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3659                                     ring_nr, i, bp->rx_ring_size);
3660                         break;
3661                 }
3662                 prod = NEXT_RX_AGG(prod);
3663         }
3664         rxr->rx_agg_prod = prod;
3665
3666         if (rxr->rx_tpa) {
3667                 dma_addr_t mapping;
3668                 u8 *data;
3669
3670                 for (i = 0; i < bp->max_tpa; i++) {
3671                         data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
3672                         if (!data)
3673                                 return -ENOMEM;
3674
3675                         rxr->rx_tpa[i].data = data;
3676                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3677                         rxr->rx_tpa[i].mapping = mapping;
3678                 }
3679         }
3680         return 0;
3681 }
3682
3683 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3684 {
3685         struct bnxt_rx_ring_info *rxr;
3686         struct bnxt_ring_struct *ring;
3687         u32 type;
3688
3689         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3690                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3691
3692         if (NET_IP_ALIGN == 2)
3693                 type |= RX_BD_FLAGS_SOP;
3694
3695         rxr = &bp->rx_ring[ring_nr];
3696         ring = &rxr->rx_ring_struct;
3697         bnxt_init_rxbd_pages(ring, type);
3698
3699         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3700                 bpf_prog_add(bp->xdp_prog, 1);
3701                 rxr->xdp_prog = bp->xdp_prog;
3702         }
3703         ring->fw_ring_id = INVALID_HW_RING_ID;
3704
3705         ring = &rxr->rx_agg_ring_struct;
3706         ring->fw_ring_id = INVALID_HW_RING_ID;
3707
3708         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3709                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3710                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3711
3712                 bnxt_init_rxbd_pages(ring, type);
3713         }
3714
3715         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3716 }
3717
3718 static void bnxt_init_cp_rings(struct bnxt *bp)
3719 {
3720         int i, j;
3721
3722         for (i = 0; i < bp->cp_nr_rings; i++) {
3723                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3724                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3725
3726                 ring->fw_ring_id = INVALID_HW_RING_ID;
3727                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3728                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3729                 for (j = 0; j < 2; j++) {
3730                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3731
3732                         if (!cpr2)
3733                                 continue;
3734
3735                         ring = &cpr2->cp_ring_struct;
3736                         ring->fw_ring_id = INVALID_HW_RING_ID;
3737                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3738                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3739                 }
3740         }
3741 }
3742
3743 static int bnxt_init_rx_rings(struct bnxt *bp)
3744 {
3745         int i, rc = 0;
3746
3747         if (BNXT_RX_PAGE_MODE(bp)) {
3748                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3749                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3750         } else {
3751                 bp->rx_offset = BNXT_RX_OFFSET;
3752                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3753         }
3754
3755         for (i = 0; i < bp->rx_nr_rings; i++) {
3756                 rc = bnxt_init_one_rx_ring(bp, i);
3757                 if (rc)
3758                         break;
3759         }
3760
3761         return rc;
3762 }
3763
3764 static int bnxt_init_tx_rings(struct bnxt *bp)
3765 {
3766         u16 i;
3767
3768         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3769                                    BNXT_MIN_TX_DESC_CNT);
3770
3771         for (i = 0; i < bp->tx_nr_rings; i++) {
3772                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3773                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3774
3775                 ring->fw_ring_id = INVALID_HW_RING_ID;
3776         }
3777
3778         return 0;
3779 }
3780
3781 static void bnxt_free_ring_grps(struct bnxt *bp)
3782 {
3783         kfree(bp->grp_info);
3784         bp->grp_info = NULL;
3785 }
3786
3787 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3788 {
3789         int i;
3790
3791         if (irq_re_init) {
3792                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3793                                        sizeof(struct bnxt_ring_grp_info),
3794                                        GFP_KERNEL);
3795                 if (!bp->grp_info)
3796                         return -ENOMEM;
3797         }
3798         for (i = 0; i < bp->cp_nr_rings; i++) {
3799                 if (irq_re_init)
3800                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3801                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3802                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3803                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3804                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3805         }
3806         return 0;
3807 }
3808
3809 static void bnxt_free_vnics(struct bnxt *bp)
3810 {
3811         kfree(bp->vnic_info);
3812         bp->vnic_info = NULL;
3813         bp->nr_vnics = 0;
3814 }
3815
3816 static int bnxt_alloc_vnics(struct bnxt *bp)
3817 {
3818         int num_vnics = 1;
3819
3820 #ifdef CONFIG_RFS_ACCEL
3821         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3822                 num_vnics += bp->rx_nr_rings;
3823 #endif
3824
3825         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3826                 num_vnics++;
3827
3828         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3829                                 GFP_KERNEL);
3830         if (!bp->vnic_info)
3831                 return -ENOMEM;
3832
3833         bp->nr_vnics = num_vnics;
3834         return 0;
3835 }
3836
3837 static void bnxt_init_vnics(struct bnxt *bp)
3838 {
3839         int i;
3840
3841         for (i = 0; i < bp->nr_vnics; i++) {
3842                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3843                 int j;
3844
3845                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3846                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3847                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3848
3849                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3850
3851                 if (bp->vnic_info[i].rss_hash_key) {
3852                         if (i == 0)
3853                                 get_random_bytes(vnic->rss_hash_key,
3854                                               HW_HASH_KEY_SIZE);
3855                         else
3856                                 memcpy(vnic->rss_hash_key,
3857                                        bp->vnic_info[0].rss_hash_key,
3858                                        HW_HASH_KEY_SIZE);
3859                 }
3860         }
3861 }
3862
3863 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3864 {
3865         int pages;
3866
3867         pages = ring_size / desc_per_pg;
3868
3869         if (!pages)
3870                 return 1;
3871
3872         pages++;
3873
3874         while (pages & (pages - 1))
3875                 pages++;
3876
3877         return pages;
3878 }
3879
3880 void bnxt_set_tpa_flags(struct bnxt *bp)
3881 {
3882         bp->flags &= ~BNXT_FLAG_TPA;
3883         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3884                 return;
3885         if (bp->dev->features & NETIF_F_LRO)
3886                 bp->flags |= BNXT_FLAG_LRO;
3887         else if (bp->dev->features & NETIF_F_GRO_HW)
3888                 bp->flags |= BNXT_FLAG_GRO;
3889 }
3890
3891 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3892  * be set on entry.
3893  */
3894 void bnxt_set_ring_params(struct bnxt *bp)
3895 {
3896         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3897         u32 agg_factor = 0, agg_ring_size = 0;
3898
3899         /* 8 for CRC and VLAN */
3900         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3901
3902         rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
3903                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3904
3905         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3906         ring_size = bp->rx_ring_size;
3907         bp->rx_agg_ring_size = 0;
3908         bp->rx_agg_nr_pages = 0;
3909
3910         if (bp->flags & BNXT_FLAG_TPA)
3911                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3912
3913         bp->flags &= ~BNXT_FLAG_JUMBO;
3914         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3915                 u32 jumbo_factor;
3916
3917                 bp->flags |= BNXT_FLAG_JUMBO;
3918                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3919                 if (jumbo_factor > agg_factor)
3920                         agg_factor = jumbo_factor;
3921         }
3922         if (agg_factor) {
3923                 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
3924                         ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
3925                         netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
3926                                     bp->rx_ring_size, ring_size);
3927                         bp->rx_ring_size = ring_size;
3928                 }
3929                 agg_ring_size = ring_size * agg_factor;
3930
3931                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3932                                                         RX_DESC_CNT);
3933                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3934                         u32 tmp = agg_ring_size;
3935
3936                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3937                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3938                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3939                                     tmp, agg_ring_size);
3940                 }
3941                 bp->rx_agg_ring_size = agg_ring_size;
3942                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3943
3944                 if (BNXT_RX_PAGE_MODE(bp)) {
3945                         rx_space = PAGE_SIZE;
3946                         rx_size = PAGE_SIZE -
3947                                   ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
3948                                   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3949                 } else {
3950                         rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3951                         rx_space = rx_size + NET_SKB_PAD +
3952                                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3953                 }
3954         }
3955
3956         bp->rx_buf_use_size = rx_size;
3957         bp->rx_buf_size = rx_space;
3958
3959         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3960         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3961
3962         ring_size = bp->tx_ring_size;
3963         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3964         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3965
3966         max_rx_cmpl = bp->rx_ring_size;
3967         /* MAX TPA needs to be added because TPA_START completions are
3968          * immediately recycled, so the TPA completions are not bound by
3969          * the RX ring size.
3970          */
3971         if (bp->flags & BNXT_FLAG_TPA)
3972                 max_rx_cmpl += bp->max_tpa;
3973         /* RX and TPA completions are 32-byte, all others are 16-byte */
3974         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3975         bp->cp_ring_size = ring_size;
3976
3977         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3978         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3979                 bp->cp_nr_pages = MAX_CP_PAGES;
3980                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3981                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3982                             ring_size, bp->cp_ring_size);
3983         }
3984         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3985         bp->cp_ring_mask = bp->cp_bit - 1;
3986 }
3987
3988 /* Changing allocation mode of RX rings.
3989  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3990  */
3991 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3992 {
3993         if (page_mode) {
3994                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3995                 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
3996
3997                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
3998                         bp->flags |= BNXT_FLAG_JUMBO;
3999                         bp->rx_skb_func = bnxt_rx_multi_page_skb;
4000                         bp->dev->max_mtu =
4001                                 min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4002                 } else {
4003                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4004                         bp->rx_skb_func = bnxt_rx_page_skb;
4005                         bp->dev->max_mtu =
4006                                 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4007                 }
4008                 bp->rx_dir = DMA_BIDIRECTIONAL;
4009                 /* Disable LRO or GRO_HW */
4010                 netdev_update_features(bp->dev);
4011         } else {
4012                 bp->dev->max_mtu = bp->max_mtu;
4013                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4014                 bp->rx_dir = DMA_FROM_DEVICE;
4015                 bp->rx_skb_func = bnxt_rx_skb;
4016         }
4017         return 0;
4018 }
4019
4020 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4021 {
4022         int i;
4023         struct bnxt_vnic_info *vnic;
4024         struct pci_dev *pdev = bp->pdev;
4025
4026         if (!bp->vnic_info)
4027                 return;
4028
4029         for (i = 0; i < bp->nr_vnics; i++) {
4030                 vnic = &bp->vnic_info[i];
4031
4032                 kfree(vnic->fw_grp_ids);
4033                 vnic->fw_grp_ids = NULL;
4034
4035                 kfree(vnic->uc_list);
4036                 vnic->uc_list = NULL;
4037
4038                 if (vnic->mc_list) {
4039                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4040                                           vnic->mc_list, vnic->mc_list_mapping);
4041                         vnic->mc_list = NULL;
4042                 }
4043
4044                 if (vnic->rss_table) {
4045                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4046                                           vnic->rss_table,
4047                                           vnic->rss_table_dma_addr);
4048                         vnic->rss_table = NULL;
4049                 }
4050
4051                 vnic->rss_hash_key = NULL;
4052                 vnic->flags = 0;
4053         }
4054 }
4055
4056 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4057 {
4058         int i, rc = 0, size;
4059         struct bnxt_vnic_info *vnic;
4060         struct pci_dev *pdev = bp->pdev;
4061         int max_rings;
4062
4063         for (i = 0; i < bp->nr_vnics; i++) {
4064                 vnic = &bp->vnic_info[i];
4065
4066                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4067                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4068
4069                         if (mem_size > 0) {
4070                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4071                                 if (!vnic->uc_list) {
4072                                         rc = -ENOMEM;
4073                                         goto out;
4074                                 }
4075                         }
4076                 }
4077
4078                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4079                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4080                         vnic->mc_list =
4081                                 dma_alloc_coherent(&pdev->dev,
4082                                                    vnic->mc_list_size,
4083                                                    &vnic->mc_list_mapping,
4084                                                    GFP_KERNEL);
4085                         if (!vnic->mc_list) {
4086                                 rc = -ENOMEM;
4087                                 goto out;
4088                         }
4089                 }
4090
4091                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4092                         goto vnic_skip_grps;
4093
4094                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4095                         max_rings = bp->rx_nr_rings;
4096                 else
4097                         max_rings = 1;
4098
4099                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4100                 if (!vnic->fw_grp_ids) {
4101                         rc = -ENOMEM;
4102                         goto out;
4103                 }
4104 vnic_skip_grps:
4105                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
4106                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4107                         continue;
4108
4109                 /* Allocate rss table and hash key */
4110                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4111                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4112                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4113
4114                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4115                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4116                                                      vnic->rss_table_size,
4117                                                      &vnic->rss_table_dma_addr,
4118                                                      GFP_KERNEL);
4119                 if (!vnic->rss_table) {
4120                         rc = -ENOMEM;
4121                         goto out;
4122                 }
4123
4124                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4125                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4126         }
4127         return 0;
4128
4129 out:
4130         return rc;
4131 }
4132
4133 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4134 {
4135         struct bnxt_hwrm_wait_token *token;
4136
4137         dma_pool_destroy(bp->hwrm_dma_pool);
4138         bp->hwrm_dma_pool = NULL;
4139
4140         rcu_read_lock();
4141         hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4142                 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4143         rcu_read_unlock();
4144 }
4145
4146 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4147 {
4148         bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4149                                             BNXT_HWRM_DMA_SIZE,
4150                                             BNXT_HWRM_DMA_ALIGN, 0);
4151         if (!bp->hwrm_dma_pool)
4152                 return -ENOMEM;
4153
4154         INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4155
4156         return 0;
4157 }
4158
4159 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4160 {
4161         kfree(stats->hw_masks);
4162         stats->hw_masks = NULL;
4163         kfree(stats->sw_stats);
4164         stats->sw_stats = NULL;
4165         if (stats->hw_stats) {
4166                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4167                                   stats->hw_stats_map);
4168                 stats->hw_stats = NULL;
4169         }
4170 }
4171
4172 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4173                                 bool alloc_masks)
4174 {
4175         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4176                                              &stats->hw_stats_map, GFP_KERNEL);
4177         if (!stats->hw_stats)
4178                 return -ENOMEM;
4179
4180         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4181         if (!stats->sw_stats)
4182                 goto stats_mem_err;
4183
4184         if (alloc_masks) {
4185                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4186                 if (!stats->hw_masks)
4187                         goto stats_mem_err;
4188         }
4189         return 0;
4190
4191 stats_mem_err:
4192         bnxt_free_stats_mem(bp, stats);
4193         return -ENOMEM;
4194 }
4195
4196 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4197 {
4198         int i;
4199
4200         for (i = 0; i < count; i++)
4201                 mask_arr[i] = mask;
4202 }
4203
4204 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4205 {
4206         int i;
4207
4208         for (i = 0; i < count; i++)
4209                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4210 }
4211
4212 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4213                                     struct bnxt_stats_mem *stats)
4214 {
4215         struct hwrm_func_qstats_ext_output *resp;
4216         struct hwrm_func_qstats_ext_input *req;
4217         __le64 *hw_masks;
4218         int rc;
4219
4220         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4221             !(bp->flags & BNXT_FLAG_CHIP_P5))
4222                 return -EOPNOTSUPP;
4223
4224         rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4225         if (rc)
4226                 return rc;
4227
4228         req->fid = cpu_to_le16(0xffff);
4229         req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4230
4231         resp = hwrm_req_hold(bp, req);
4232         rc = hwrm_req_send(bp, req);
4233         if (!rc) {
4234                 hw_masks = &resp->rx_ucast_pkts;
4235                 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4236         }
4237         hwrm_req_drop(bp, req);
4238         return rc;
4239 }
4240
4241 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4242 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4243
4244 static void bnxt_init_stats(struct bnxt *bp)
4245 {
4246         struct bnxt_napi *bnapi = bp->bnapi[0];
4247         struct bnxt_cp_ring_info *cpr;
4248         struct bnxt_stats_mem *stats;
4249         __le64 *rx_stats, *tx_stats;
4250         int rc, rx_count, tx_count;
4251         u64 *rx_masks, *tx_masks;
4252         u64 mask;
4253         u8 flags;
4254
4255         cpr = &bnapi->cp_ring;
4256         stats = &cpr->stats;
4257         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4258         if (rc) {
4259                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4260                         mask = (1ULL << 48) - 1;
4261                 else
4262                         mask = -1ULL;
4263                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4264         }
4265         if (bp->flags & BNXT_FLAG_PORT_STATS) {
4266                 stats = &bp->port_stats;
4267                 rx_stats = stats->hw_stats;
4268                 rx_masks = stats->hw_masks;
4269                 rx_count = sizeof(struct rx_port_stats) / 8;
4270                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4271                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4272                 tx_count = sizeof(struct tx_port_stats) / 8;
4273
4274                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4275                 rc = bnxt_hwrm_port_qstats(bp, flags);
4276                 if (rc) {
4277                         mask = (1ULL << 40) - 1;
4278
4279                         bnxt_fill_masks(rx_masks, mask, rx_count);
4280                         bnxt_fill_masks(tx_masks, mask, tx_count);
4281                 } else {
4282                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4283                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4284                         bnxt_hwrm_port_qstats(bp, 0);
4285                 }
4286         }
4287         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4288                 stats = &bp->rx_port_stats_ext;
4289                 rx_stats = stats->hw_stats;
4290                 rx_masks = stats->hw_masks;
4291                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4292                 stats = &bp->tx_port_stats_ext;
4293                 tx_stats = stats->hw_stats;
4294                 tx_masks = stats->hw_masks;
4295                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4296
4297                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4298                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4299                 if (rc) {
4300                         mask = (1ULL << 40) - 1;
4301
4302                         bnxt_fill_masks(rx_masks, mask, rx_count);
4303                         if (tx_stats)
4304                                 bnxt_fill_masks(tx_masks, mask, tx_count);
4305                 } else {
4306                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4307                         if (tx_stats)
4308                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
4309                                                    tx_count);
4310                         bnxt_hwrm_port_qstats_ext(bp, 0);
4311                 }
4312         }
4313 }
4314
4315 static void bnxt_free_port_stats(struct bnxt *bp)
4316 {
4317         bp->flags &= ~BNXT_FLAG_PORT_STATS;
4318         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4319
4320         bnxt_free_stats_mem(bp, &bp->port_stats);
4321         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4322         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4323 }
4324
4325 static void bnxt_free_ring_stats(struct bnxt *bp)
4326 {
4327         int i;
4328
4329         if (!bp->bnapi)
4330                 return;
4331
4332         for (i = 0; i < bp->cp_nr_rings; i++) {
4333                 struct bnxt_napi *bnapi = bp->bnapi[i];
4334                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4335
4336                 bnxt_free_stats_mem(bp, &cpr->stats);
4337         }
4338 }
4339
4340 static int bnxt_alloc_stats(struct bnxt *bp)
4341 {
4342         u32 size, i;
4343         int rc;
4344
4345         size = bp->hw_ring_stats_size;
4346
4347         for (i = 0; i < bp->cp_nr_rings; i++) {
4348                 struct bnxt_napi *bnapi = bp->bnapi[i];
4349                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4350
4351                 cpr->stats.len = size;
4352                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4353                 if (rc)
4354                         return rc;
4355
4356                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4357         }
4358
4359         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4360                 return 0;
4361
4362         if (bp->port_stats.hw_stats)
4363                 goto alloc_ext_stats;
4364
4365         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4366         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4367         if (rc)
4368                 return rc;
4369
4370         bp->flags |= BNXT_FLAG_PORT_STATS;
4371
4372 alloc_ext_stats:
4373         /* Display extended statistics only if FW supports it */
4374         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4375                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4376                         return 0;
4377
4378         if (bp->rx_port_stats_ext.hw_stats)
4379                 goto alloc_tx_ext_stats;
4380
4381         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4382         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4383         /* Extended stats are optional */
4384         if (rc)
4385                 return 0;
4386
4387 alloc_tx_ext_stats:
4388         if (bp->tx_port_stats_ext.hw_stats)
4389                 return 0;
4390
4391         if (bp->hwrm_spec_code >= 0x10902 ||
4392             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4393                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4394                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4395                 /* Extended stats are optional */
4396                 if (rc)
4397                         return 0;
4398         }
4399         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4400         return 0;
4401 }
4402
4403 static void bnxt_clear_ring_indices(struct bnxt *bp)
4404 {
4405         int i;
4406
4407         if (!bp->bnapi)
4408                 return;
4409
4410         for (i = 0; i < bp->cp_nr_rings; i++) {
4411                 struct bnxt_napi *bnapi = bp->bnapi[i];
4412                 struct bnxt_cp_ring_info *cpr;
4413                 struct bnxt_rx_ring_info *rxr;
4414                 struct bnxt_tx_ring_info *txr;
4415
4416                 if (!bnapi)
4417                         continue;
4418
4419                 cpr = &bnapi->cp_ring;
4420                 cpr->cp_raw_cons = 0;
4421
4422                 txr = bnapi->tx_ring;
4423                 if (txr) {
4424                         txr->tx_prod = 0;
4425                         txr->tx_cons = 0;
4426                 }
4427
4428                 rxr = bnapi->rx_ring;
4429                 if (rxr) {
4430                         rxr->rx_prod = 0;
4431                         rxr->rx_agg_prod = 0;
4432                         rxr->rx_sw_agg_prod = 0;
4433                         rxr->rx_next_cons = 0;
4434                 }
4435         }
4436 }
4437
4438 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4439 {
4440 #ifdef CONFIG_RFS_ACCEL
4441         int i;
4442
4443         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4444          * safe to delete the hash table.
4445          */
4446         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4447                 struct hlist_head *head;
4448                 struct hlist_node *tmp;
4449                 struct bnxt_ntuple_filter *fltr;
4450
4451                 head = &bp->ntp_fltr_hash_tbl[i];
4452                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4453                         hlist_del(&fltr->hash);
4454                         kfree(fltr);
4455                 }
4456         }
4457         if (irq_reinit) {
4458                 bitmap_free(bp->ntp_fltr_bmap);
4459                 bp->ntp_fltr_bmap = NULL;
4460         }
4461         bp->ntp_fltr_count = 0;
4462 #endif
4463 }
4464
4465 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4466 {
4467 #ifdef CONFIG_RFS_ACCEL
4468         int i, rc = 0;
4469
4470         if (!(bp->flags & BNXT_FLAG_RFS))
4471                 return 0;
4472
4473         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4474                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4475
4476         bp->ntp_fltr_count = 0;
4477         bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL);
4478
4479         if (!bp->ntp_fltr_bmap)
4480                 rc = -ENOMEM;
4481
4482         return rc;
4483 #else
4484         return 0;
4485 #endif
4486 }
4487
4488 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4489 {
4490         bnxt_free_vnic_attributes(bp);
4491         bnxt_free_tx_rings(bp);
4492         bnxt_free_rx_rings(bp);
4493         bnxt_free_cp_rings(bp);
4494         bnxt_free_all_cp_arrays(bp);
4495         bnxt_free_ntp_fltrs(bp, irq_re_init);
4496         if (irq_re_init) {
4497                 bnxt_free_ring_stats(bp);
4498                 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4499                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4500                         bnxt_free_port_stats(bp);
4501                 bnxt_free_ring_grps(bp);
4502                 bnxt_free_vnics(bp);
4503                 kfree(bp->tx_ring_map);
4504                 bp->tx_ring_map = NULL;
4505                 kfree(bp->tx_ring);
4506                 bp->tx_ring = NULL;
4507                 kfree(bp->rx_ring);
4508                 bp->rx_ring = NULL;
4509                 kfree(bp->bnapi);
4510                 bp->bnapi = NULL;
4511         } else {
4512                 bnxt_clear_ring_indices(bp);
4513         }
4514 }
4515
4516 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4517 {
4518         int i, j, rc, size, arr_size;
4519         void *bnapi;
4520
4521         if (irq_re_init) {
4522                 /* Allocate bnapi mem pointer array and mem block for
4523                  * all queues
4524                  */
4525                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4526                                 bp->cp_nr_rings);
4527                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4528                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4529                 if (!bnapi)
4530                         return -ENOMEM;
4531
4532                 bp->bnapi = bnapi;
4533                 bnapi += arr_size;
4534                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4535                         bp->bnapi[i] = bnapi;
4536                         bp->bnapi[i]->index = i;
4537                         bp->bnapi[i]->bp = bp;
4538                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4539                                 struct bnxt_cp_ring_info *cpr =
4540                                         &bp->bnapi[i]->cp_ring;
4541
4542                                 cpr->cp_ring_struct.ring_mem.flags =
4543                                         BNXT_RMEM_RING_PTE_FLAG;
4544                         }
4545                 }
4546
4547                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4548                                       sizeof(struct bnxt_rx_ring_info),
4549                                       GFP_KERNEL);
4550                 if (!bp->rx_ring)
4551                         return -ENOMEM;
4552
4553                 for (i = 0; i < bp->rx_nr_rings; i++) {
4554                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4555
4556                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4557                                 rxr->rx_ring_struct.ring_mem.flags =
4558                                         BNXT_RMEM_RING_PTE_FLAG;
4559                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4560                                         BNXT_RMEM_RING_PTE_FLAG;
4561                         }
4562                         rxr->bnapi = bp->bnapi[i];
4563                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4564                 }
4565
4566                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4567                                       sizeof(struct bnxt_tx_ring_info),
4568                                       GFP_KERNEL);
4569                 if (!bp->tx_ring)
4570                         return -ENOMEM;
4571
4572                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4573                                           GFP_KERNEL);
4574
4575                 if (!bp->tx_ring_map)
4576                         return -ENOMEM;
4577
4578                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4579                         j = 0;
4580                 else
4581                         j = bp->rx_nr_rings;
4582
4583                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4584                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4585
4586                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4587                                 txr->tx_ring_struct.ring_mem.flags =
4588                                         BNXT_RMEM_RING_PTE_FLAG;
4589                         txr->bnapi = bp->bnapi[j];
4590                         bp->bnapi[j]->tx_ring = txr;
4591                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4592                         if (i >= bp->tx_nr_rings_xdp) {
4593                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4594                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4595                         } else {
4596                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4597                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4598                         }
4599                 }
4600
4601                 rc = bnxt_alloc_stats(bp);
4602                 if (rc)
4603                         goto alloc_mem_err;
4604                 bnxt_init_stats(bp);
4605
4606                 rc = bnxt_alloc_ntp_fltrs(bp);
4607                 if (rc)
4608                         goto alloc_mem_err;
4609
4610                 rc = bnxt_alloc_vnics(bp);
4611                 if (rc)
4612                         goto alloc_mem_err;
4613         }
4614
4615         rc = bnxt_alloc_all_cp_arrays(bp);
4616         if (rc)
4617                 goto alloc_mem_err;
4618
4619         bnxt_init_ring_struct(bp);
4620
4621         rc = bnxt_alloc_rx_rings(bp);
4622         if (rc)
4623                 goto alloc_mem_err;
4624
4625         rc = bnxt_alloc_tx_rings(bp);
4626         if (rc)
4627                 goto alloc_mem_err;
4628
4629         rc = bnxt_alloc_cp_rings(bp);
4630         if (rc)
4631                 goto alloc_mem_err;
4632
4633         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4634                                   BNXT_VNIC_UCAST_FLAG;
4635         rc = bnxt_alloc_vnic_attributes(bp);
4636         if (rc)
4637                 goto alloc_mem_err;
4638         return 0;
4639
4640 alloc_mem_err:
4641         bnxt_free_mem(bp, true);
4642         return rc;
4643 }
4644
4645 static void bnxt_disable_int(struct bnxt *bp)
4646 {
4647         int i;
4648
4649         if (!bp->bnapi)
4650                 return;
4651
4652         for (i = 0; i < bp->cp_nr_rings; i++) {
4653                 struct bnxt_napi *bnapi = bp->bnapi[i];
4654                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4655                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4656
4657                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4658                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4659         }
4660 }
4661
4662 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4663 {
4664         struct bnxt_napi *bnapi = bp->bnapi[n];
4665         struct bnxt_cp_ring_info *cpr;
4666
4667         cpr = &bnapi->cp_ring;
4668         return cpr->cp_ring_struct.map_idx;
4669 }
4670
4671 static void bnxt_disable_int_sync(struct bnxt *bp)
4672 {
4673         int i;
4674
4675         if (!bp->irq_tbl)
4676                 return;
4677
4678         atomic_inc(&bp->intr_sem);
4679
4680         bnxt_disable_int(bp);
4681         for (i = 0; i < bp->cp_nr_rings; i++) {
4682                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4683
4684                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4685         }
4686 }
4687
4688 static void bnxt_enable_int(struct bnxt *bp)
4689 {
4690         int i;
4691
4692         atomic_set(&bp->intr_sem, 0);
4693         for (i = 0; i < bp->cp_nr_rings; i++) {
4694                 struct bnxt_napi *bnapi = bp->bnapi[i];
4695                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4696
4697                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4698         }
4699 }
4700
4701 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4702                             bool async_only)
4703 {
4704         DECLARE_BITMAP(async_events_bmap, 256);
4705         u32 *events = (u32 *)async_events_bmap;
4706         struct hwrm_func_drv_rgtr_output *resp;
4707         struct hwrm_func_drv_rgtr_input *req;
4708         u32 flags;
4709         int rc, i;
4710
4711         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
4712         if (rc)
4713                 return rc;
4714
4715         req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4716                                    FUNC_DRV_RGTR_REQ_ENABLES_VER |
4717                                    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4718
4719         req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4720         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4721         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4722                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4723         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4724                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4725                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4726         req->flags = cpu_to_le32(flags);
4727         req->ver_maj_8b = DRV_VER_MAJ;
4728         req->ver_min_8b = DRV_VER_MIN;
4729         req->ver_upd_8b = DRV_VER_UPD;
4730         req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
4731         req->ver_min = cpu_to_le16(DRV_VER_MIN);
4732         req->ver_upd = cpu_to_le16(DRV_VER_UPD);
4733
4734         if (BNXT_PF(bp)) {
4735                 u32 data[8];
4736                 int i;
4737
4738                 memset(data, 0, sizeof(data));
4739                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4740                         u16 cmd = bnxt_vf_req_snif[i];
4741                         unsigned int bit, idx;
4742
4743                         idx = cmd / 32;
4744                         bit = cmd % 32;
4745                         data[idx] |= 1 << bit;
4746                 }
4747
4748                 for (i = 0; i < 8; i++)
4749                         req->vf_req_fwd[i] = cpu_to_le32(data[i]);
4750
4751                 req->enables |=
4752                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4753         }
4754
4755         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4756                 req->flags |= cpu_to_le32(
4757                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4758
4759         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4760         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4761                 u16 event_id = bnxt_async_events_arr[i];
4762
4763                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4764                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4765                         continue;
4766                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4767         }
4768         if (bmap && bmap_size) {
4769                 for (i = 0; i < bmap_size; i++) {
4770                         if (test_bit(i, bmap))
4771                                 __set_bit(i, async_events_bmap);
4772                 }
4773         }
4774         for (i = 0; i < 8; i++)
4775                 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
4776
4777         if (async_only)
4778                 req->enables =
4779                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4780
4781         resp = hwrm_req_hold(bp, req);
4782         rc = hwrm_req_send(bp, req);
4783         if (!rc) {
4784                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4785                 if (resp->flags &
4786                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4787                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4788         }
4789         hwrm_req_drop(bp, req);
4790         return rc;
4791 }
4792
4793 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4794 {
4795         struct hwrm_func_drv_unrgtr_input *req;
4796         int rc;
4797
4798         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4799                 return 0;
4800
4801         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
4802         if (rc)
4803                 return rc;
4804         return hwrm_req_send(bp, req);
4805 }
4806
4807 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4808 {
4809         struct hwrm_tunnel_dst_port_free_input *req;
4810         int rc;
4811
4812         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
4813             bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
4814                 return 0;
4815         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
4816             bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
4817                 return 0;
4818
4819         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
4820         if (rc)
4821                 return rc;
4822
4823         req->tunnel_type = tunnel_type;
4824
4825         switch (tunnel_type) {
4826         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4827                 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4828                 bp->vxlan_port = 0;
4829                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4830                 break;
4831         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4832                 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4833                 bp->nge_port = 0;
4834                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4835                 break;
4836         default:
4837                 break;
4838         }
4839
4840         rc = hwrm_req_send(bp, req);
4841         if (rc)
4842                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4843                            rc);
4844         return rc;
4845 }
4846
4847 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4848                                            u8 tunnel_type)
4849 {
4850         struct hwrm_tunnel_dst_port_alloc_output *resp;
4851         struct hwrm_tunnel_dst_port_alloc_input *req;
4852         int rc;
4853
4854         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
4855         if (rc)
4856                 return rc;
4857
4858         req->tunnel_type = tunnel_type;
4859         req->tunnel_dst_port_val = port;
4860
4861         resp = hwrm_req_hold(bp, req);
4862         rc = hwrm_req_send(bp, req);
4863         if (rc) {
4864                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4865                            rc);
4866                 goto err_out;
4867         }
4868
4869         switch (tunnel_type) {
4870         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4871                 bp->vxlan_port = port;
4872                 bp->vxlan_fw_dst_port_id =
4873                         le16_to_cpu(resp->tunnel_dst_port_id);
4874                 break;
4875         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4876                 bp->nge_port = port;
4877                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4878                 break;
4879         default:
4880                 break;
4881         }
4882
4883 err_out:
4884         hwrm_req_drop(bp, req);
4885         return rc;
4886 }
4887
4888 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4889 {
4890         struct hwrm_cfa_l2_set_rx_mask_input *req;
4891         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4892         int rc;
4893
4894         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
4895         if (rc)
4896                 return rc;
4897
4898         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4899         if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
4900                 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4901                 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4902         }
4903         req->mask = cpu_to_le32(vnic->rx_mask);
4904         return hwrm_req_send_silent(bp, req);
4905 }
4906
4907 #ifdef CONFIG_RFS_ACCEL
4908 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4909                                             struct bnxt_ntuple_filter *fltr)
4910 {
4911         struct hwrm_cfa_ntuple_filter_free_input *req;
4912         int rc;
4913
4914         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
4915         if (rc)
4916                 return rc;
4917
4918         req->ntuple_filter_id = fltr->filter_id;
4919         return hwrm_req_send(bp, req);
4920 }
4921
4922 #define BNXT_NTP_FLTR_FLAGS                                     \
4923         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4924          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4925          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4926          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4927          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4928          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4929          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4930          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4931          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4932          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4933          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4934          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4935          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4936          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4937
4938 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4939                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4940
4941 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4942                                              struct bnxt_ntuple_filter *fltr)
4943 {
4944         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4945         struct hwrm_cfa_ntuple_filter_alloc_input *req;
4946         struct flow_keys *keys = &fltr->fkeys;
4947         struct bnxt_vnic_info *vnic;
4948         u32 flags = 0;
4949         int rc;
4950
4951         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
4952         if (rc)
4953                 return rc;
4954
4955         req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4956
4957         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4958                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4959                 req->dst_id = cpu_to_le16(fltr->rxq);
4960         } else {
4961                 vnic = &bp->vnic_info[fltr->rxq + 1];
4962                 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
4963         }
4964         req->flags = cpu_to_le32(flags);
4965         req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4966
4967         req->ethertype = htons(ETH_P_IP);
4968         memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4969         req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4970         req->ip_protocol = keys->basic.ip_proto;
4971
4972         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4973                 int i;
4974
4975                 req->ethertype = htons(ETH_P_IPV6);
4976                 req->ip_addr_type =
4977                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4978                 *(struct in6_addr *)&req->src_ipaddr[0] =
4979                         keys->addrs.v6addrs.src;
4980                 *(struct in6_addr *)&req->dst_ipaddr[0] =
4981                         keys->addrs.v6addrs.dst;
4982                 for (i = 0; i < 4; i++) {
4983                         req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4984                         req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4985                 }
4986         } else {
4987                 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
4988                 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4989                 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4990                 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4991         }
4992         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4993                 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4994                 req->tunnel_type =
4995                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4996         }
4997
4998         req->src_port = keys->ports.src;
4999         req->src_port_mask = cpu_to_be16(0xffff);
5000         req->dst_port = keys->ports.dst;
5001         req->dst_port_mask = cpu_to_be16(0xffff);
5002
5003         resp = hwrm_req_hold(bp, req);
5004         rc = hwrm_req_send(bp, req);
5005         if (!rc)
5006                 fltr->filter_id = resp->ntuple_filter_id;
5007         hwrm_req_drop(bp, req);
5008         return rc;
5009 }
5010 #endif
5011
5012 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5013                                      const u8 *mac_addr)
5014 {
5015         struct hwrm_cfa_l2_filter_alloc_output *resp;
5016         struct hwrm_cfa_l2_filter_alloc_input *req;
5017         int rc;
5018
5019         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5020         if (rc)
5021                 return rc;
5022
5023         req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5024         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5025                 req->flags |=
5026                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5027         req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
5028         req->enables =
5029                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5030                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5031                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5032         memcpy(req->l2_addr, mac_addr, ETH_ALEN);
5033         req->l2_addr_mask[0] = 0xff;
5034         req->l2_addr_mask[1] = 0xff;
5035         req->l2_addr_mask[2] = 0xff;
5036         req->l2_addr_mask[3] = 0xff;
5037         req->l2_addr_mask[4] = 0xff;
5038         req->l2_addr_mask[5] = 0xff;
5039
5040         resp = hwrm_req_hold(bp, req);
5041         rc = hwrm_req_send(bp, req);
5042         if (!rc)
5043                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
5044                                                         resp->l2_filter_id;
5045         hwrm_req_drop(bp, req);
5046         return rc;
5047 }
5048
5049 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5050 {
5051         struct hwrm_cfa_l2_filter_free_input *req;
5052         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5053         int rc;
5054
5055         /* Any associated ntuple filters will also be cleared by firmware. */
5056         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5057         if (rc)
5058                 return rc;
5059         hwrm_req_hold(bp, req);
5060         for (i = 0; i < num_of_vnics; i++) {
5061                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5062
5063                 for (j = 0; j < vnic->uc_filter_count; j++) {
5064                         req->l2_filter_id = vnic->fw_l2_filter_id[j];
5065
5066                         rc = hwrm_req_send(bp, req);
5067                 }
5068                 vnic->uc_filter_count = 0;
5069         }
5070         hwrm_req_drop(bp, req);
5071         return rc;
5072 }
5073
5074 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5075 {
5076         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5077         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5078         struct hwrm_vnic_tpa_cfg_input *req;
5079         int rc;
5080
5081         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5082                 return 0;
5083
5084         rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5085         if (rc)
5086                 return rc;
5087
5088         if (tpa_flags) {
5089                 u16 mss = bp->dev->mtu - 40;
5090                 u32 nsegs, n, segs = 0, flags;
5091
5092                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5093                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5094                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5095                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5096                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5097                 if (tpa_flags & BNXT_FLAG_GRO)
5098                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5099
5100                 req->flags = cpu_to_le32(flags);
5101
5102                 req->enables =
5103                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5104                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5105                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5106
5107                 /* Number of segs are log2 units, and first packet is not
5108                  * included as part of this units.
5109                  */
5110                 if (mss <= BNXT_RX_PAGE_SIZE) {
5111                         n = BNXT_RX_PAGE_SIZE / mss;
5112                         nsegs = (MAX_SKB_FRAGS - 1) * n;
5113                 } else {
5114                         n = mss / BNXT_RX_PAGE_SIZE;
5115                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
5116                                 n++;
5117                         nsegs = (MAX_SKB_FRAGS - n) / n;
5118                 }
5119
5120                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5121                         segs = MAX_TPA_SEGS_P5;
5122                         max_aggs = bp->max_tpa;
5123                 } else {
5124                         segs = ilog2(nsegs);
5125                 }
5126                 req->max_agg_segs = cpu_to_le16(segs);
5127                 req->max_aggs = cpu_to_le16(max_aggs);
5128
5129                 req->min_agg_len = cpu_to_le32(512);
5130         }
5131         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5132
5133         return hwrm_req_send(bp, req);
5134 }
5135
5136 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5137 {
5138         struct bnxt_ring_grp_info *grp_info;
5139
5140         grp_info = &bp->grp_info[ring->grp_idx];
5141         return grp_info->cp_fw_ring_id;
5142 }
5143
5144 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5145 {
5146         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5147                 struct bnxt_napi *bnapi = rxr->bnapi;
5148                 struct bnxt_cp_ring_info *cpr;
5149
5150                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5151                 return cpr->cp_ring_struct.fw_ring_id;
5152         } else {
5153                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5154         }
5155 }
5156
5157 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5158 {
5159         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5160                 struct bnxt_napi *bnapi = txr->bnapi;
5161                 struct bnxt_cp_ring_info *cpr;
5162
5163                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5164                 return cpr->cp_ring_struct.fw_ring_id;
5165         } else {
5166                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5167         }
5168 }
5169
5170 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5171 {
5172         int entries;
5173
5174         if (bp->flags & BNXT_FLAG_CHIP_P5)
5175                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5176         else
5177                 entries = HW_HASH_INDEX_SIZE;
5178
5179         bp->rss_indir_tbl_entries = entries;
5180         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5181                                           GFP_KERNEL);
5182         if (!bp->rss_indir_tbl)
5183                 return -ENOMEM;
5184         return 0;
5185 }
5186
5187 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5188 {
5189         u16 max_rings, max_entries, pad, i;
5190
5191         if (!bp->rx_nr_rings)
5192                 return;
5193
5194         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5195                 max_rings = bp->rx_nr_rings - 1;
5196         else
5197                 max_rings = bp->rx_nr_rings;
5198
5199         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5200
5201         for (i = 0; i < max_entries; i++)
5202                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5203
5204         pad = bp->rss_indir_tbl_entries - max_entries;
5205         if (pad)
5206                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5207 }
5208
5209 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5210 {
5211         u16 i, tbl_size, max_ring = 0;
5212
5213         if (!bp->rss_indir_tbl)
5214                 return 0;
5215
5216         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5217         for (i = 0; i < tbl_size; i++)
5218                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5219         return max_ring;
5220 }
5221
5222 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5223 {
5224         if (bp->flags & BNXT_FLAG_CHIP_P5)
5225                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5226         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5227                 return 2;
5228         return 1;
5229 }
5230
5231 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5232 {
5233         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5234         u16 i, j;
5235
5236         /* Fill the RSS indirection table with ring group ids */
5237         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5238                 if (!no_rss)
5239                         j = bp->rss_indir_tbl[i];
5240                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5241         }
5242 }
5243
5244 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5245                                     struct bnxt_vnic_info *vnic)
5246 {
5247         __le16 *ring_tbl = vnic->rss_table;
5248         struct bnxt_rx_ring_info *rxr;
5249         u16 tbl_size, i;
5250
5251         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5252
5253         for (i = 0; i < tbl_size; i++) {
5254                 u16 ring_id, j;
5255
5256                 j = bp->rss_indir_tbl[i];
5257                 rxr = &bp->rx_ring[j];
5258
5259                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5260                 *ring_tbl++ = cpu_to_le16(ring_id);
5261                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5262                 *ring_tbl++ = cpu_to_le16(ring_id);
5263         }
5264 }
5265
5266 static void
5267 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5268                          struct bnxt_vnic_info *vnic)
5269 {
5270         if (bp->flags & BNXT_FLAG_CHIP_P5)
5271                 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5272         else
5273                 bnxt_fill_hw_rss_tbl(bp, vnic);
5274
5275         if (bp->rss_hash_delta) {
5276                 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5277                 if (bp->rss_hash_cfg & bp->rss_hash_delta)
5278                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5279                 else
5280                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5281         } else {
5282                 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5283         }
5284         req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5285         req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5286         req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5287 }
5288
5289 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5290 {
5291         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5292         struct hwrm_vnic_rss_cfg_input *req;
5293         int rc;
5294
5295         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5296             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5297                 return 0;
5298
5299         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5300         if (rc)
5301                 return rc;
5302
5303         if (set_rss)
5304                 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5305         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5306         return hwrm_req_send(bp, req);
5307 }
5308
5309 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5310 {
5311         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5312         struct hwrm_vnic_rss_cfg_input *req;
5313         dma_addr_t ring_tbl_map;
5314         u32 i, nr_ctxs;
5315         int rc;
5316
5317         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5318         if (rc)
5319                 return rc;
5320
5321         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5322         if (!set_rss)
5323                 return hwrm_req_send(bp, req);
5324
5325         __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5326         ring_tbl_map = vnic->rss_table_dma_addr;
5327         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5328
5329         hwrm_req_hold(bp, req);
5330         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5331                 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5332                 req->ring_table_pair_index = i;
5333                 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5334                 rc = hwrm_req_send(bp, req);
5335                 if (rc)
5336                         goto exit;
5337         }
5338
5339 exit:
5340         hwrm_req_drop(bp, req);
5341         return rc;
5342 }
5343
5344 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
5345 {
5346         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5347         struct hwrm_vnic_rss_qcfg_output *resp;
5348         struct hwrm_vnic_rss_qcfg_input *req;
5349
5350         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
5351                 return;
5352
5353         /* all contexts configured to same hash_type, zero always exists */
5354         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5355         resp = hwrm_req_hold(bp, req);
5356         if (!hwrm_req_send(bp, req)) {
5357                 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
5358                 bp->rss_hash_delta = 0;
5359         }
5360         hwrm_req_drop(bp, req);
5361 }
5362
5363 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5364 {
5365         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5366         struct hwrm_vnic_plcmodes_cfg_input *req;
5367         int rc;
5368
5369         rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
5370         if (rc)
5371                 return rc;
5372
5373         req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
5374         req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
5375
5376         if (BNXT_RX_PAGE_MODE(bp)) {
5377                 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
5378         } else {
5379                 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5380                                           VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5381                 req->enables |=
5382                         cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5383                 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5384                 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5385         }
5386         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5387         return hwrm_req_send(bp, req);
5388 }
5389
5390 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5391                                         u16 ctx_idx)
5392 {
5393         struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
5394
5395         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
5396                 return;
5397
5398         req->rss_cos_lb_ctx_id =
5399                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5400
5401         hwrm_req_send(bp, req);
5402         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5403 }
5404
5405 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5406 {
5407         int i, j;
5408
5409         for (i = 0; i < bp->nr_vnics; i++) {
5410                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5411
5412                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5413                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5414                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5415                 }
5416         }
5417         bp->rsscos_nr_ctxs = 0;
5418 }
5419
5420 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5421 {
5422         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
5423         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
5424         int rc;
5425
5426         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
5427         if (rc)
5428                 return rc;
5429
5430         resp = hwrm_req_hold(bp, req);
5431         rc = hwrm_req_send(bp, req);
5432         if (!rc)
5433                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5434                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5435         hwrm_req_drop(bp, req);
5436
5437         return rc;
5438 }
5439
5440 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5441 {
5442         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5443                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5444         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5445 }
5446
5447 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5448 {
5449         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5450         struct hwrm_vnic_cfg_input *req;
5451         unsigned int ring = 0, grp_idx;
5452         u16 def_vlan = 0;
5453         int rc;
5454
5455         rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
5456         if (rc)
5457                 return rc;
5458
5459         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5460                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5461
5462                 req->default_rx_ring_id =
5463                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5464                 req->default_cmpl_ring_id =
5465                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5466                 req->enables =
5467                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5468                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5469                 goto vnic_mru;
5470         }
5471         req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5472         /* Only RSS support for now TBD: COS & LB */
5473         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5474                 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5475                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5476                                            VNIC_CFG_REQ_ENABLES_MRU);
5477         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5478                 req->rss_rule =
5479                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5480                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5481                                            VNIC_CFG_REQ_ENABLES_MRU);
5482                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5483         } else {
5484                 req->rss_rule = cpu_to_le16(0xffff);
5485         }
5486
5487         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5488             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5489                 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5490                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5491         } else {
5492                 req->cos_rule = cpu_to_le16(0xffff);
5493         }
5494
5495         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5496                 ring = 0;
5497         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5498                 ring = vnic_id - 1;
5499         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5500                 ring = bp->rx_nr_rings - 1;
5501
5502         grp_idx = bp->rx_ring[ring].bnapi->index;
5503         req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5504         req->lb_rule = cpu_to_le16(0xffff);
5505 vnic_mru:
5506         req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5507
5508         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5509 #ifdef CONFIG_BNXT_SRIOV
5510         if (BNXT_VF(bp))
5511                 def_vlan = bp->vf.vlan;
5512 #endif
5513         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5514                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5515         if (!vnic_id && bnxt_ulp_registered(bp->edev))
5516                 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5517
5518         return hwrm_req_send(bp, req);
5519 }
5520
5521 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5522 {
5523         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5524                 struct hwrm_vnic_free_input *req;
5525
5526                 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
5527                         return;
5528
5529                 req->vnic_id =
5530                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5531
5532                 hwrm_req_send(bp, req);
5533                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5534         }
5535 }
5536
5537 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5538 {
5539         u16 i;
5540
5541         for (i = 0; i < bp->nr_vnics; i++)
5542                 bnxt_hwrm_vnic_free_one(bp, i);
5543 }
5544
5545 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5546                                 unsigned int start_rx_ring_idx,
5547                                 unsigned int nr_rings)
5548 {
5549         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5550         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5551         struct hwrm_vnic_alloc_output *resp;
5552         struct hwrm_vnic_alloc_input *req;
5553         int rc;
5554
5555         rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
5556         if (rc)
5557                 return rc;
5558
5559         if (bp->flags & BNXT_FLAG_CHIP_P5)
5560                 goto vnic_no_ring_grps;
5561
5562         /* map ring groups to this vnic */
5563         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5564                 grp_idx = bp->rx_ring[i].bnapi->index;
5565                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5566                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5567                                    j, nr_rings);
5568                         break;
5569                 }
5570                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5571         }
5572
5573 vnic_no_ring_grps:
5574         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5575                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5576         if (vnic_id == 0)
5577                 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5578
5579         resp = hwrm_req_hold(bp, req);
5580         rc = hwrm_req_send(bp, req);
5581         if (!rc)
5582                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5583         hwrm_req_drop(bp, req);
5584         return rc;
5585 }
5586
5587 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5588 {
5589         struct hwrm_vnic_qcaps_output *resp;
5590         struct hwrm_vnic_qcaps_input *req;
5591         int rc;
5592
5593         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5594         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5595         if (bp->hwrm_spec_code < 0x10600)
5596                 return 0;
5597
5598         rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
5599         if (rc)
5600                 return rc;
5601
5602         resp = hwrm_req_hold(bp, req);
5603         rc = hwrm_req_send(bp, req);
5604         if (!rc) {
5605                 u32 flags = le32_to_cpu(resp->flags);
5606
5607                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5608                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5609                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5610                 if (flags &
5611                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5612                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5613
5614                 /* Older P5 fw before EXT_HW_STATS support did not set
5615                  * VLAN_STRIP_CAP properly.
5616                  */
5617                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5618                     (BNXT_CHIP_P5_THOR(bp) &&
5619                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5620                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5621                 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
5622                         bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA;
5623                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5624                 if (bp->max_tpa_v2) {
5625                         if (BNXT_CHIP_P5_THOR(bp))
5626                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5627                         else
5628                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5629                 }
5630         }
5631         hwrm_req_drop(bp, req);
5632         return rc;
5633 }
5634
5635 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5636 {
5637         struct hwrm_ring_grp_alloc_output *resp;
5638         struct hwrm_ring_grp_alloc_input *req;
5639         int rc;
5640         u16 i;
5641
5642         if (bp->flags & BNXT_FLAG_CHIP_P5)
5643                 return 0;
5644
5645         rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
5646         if (rc)
5647                 return rc;
5648
5649         resp = hwrm_req_hold(bp, req);
5650         for (i = 0; i < bp->rx_nr_rings; i++) {
5651                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5652
5653                 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5654                 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5655                 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5656                 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5657
5658                 rc = hwrm_req_send(bp, req);
5659
5660                 if (rc)
5661                         break;
5662
5663                 bp->grp_info[grp_idx].fw_grp_id =
5664                         le32_to_cpu(resp->ring_group_id);
5665         }
5666         hwrm_req_drop(bp, req);
5667         return rc;
5668 }
5669
5670 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5671 {
5672         struct hwrm_ring_grp_free_input *req;
5673         u16 i;
5674
5675         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5676                 return;
5677
5678         if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
5679                 return;
5680
5681         hwrm_req_hold(bp, req);
5682         for (i = 0; i < bp->cp_nr_rings; i++) {
5683                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5684                         continue;
5685                 req->ring_group_id =
5686                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5687
5688                 hwrm_req_send(bp, req);
5689                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5690         }
5691         hwrm_req_drop(bp, req);
5692 }
5693
5694 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5695                                     struct bnxt_ring_struct *ring,
5696                                     u32 ring_type, u32 map_index)
5697 {
5698         struct hwrm_ring_alloc_output *resp;
5699         struct hwrm_ring_alloc_input *req;
5700         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5701         struct bnxt_ring_grp_info *grp_info;
5702         int rc, err = 0;
5703         u16 ring_id;
5704
5705         rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
5706         if (rc)
5707                 goto exit;
5708
5709         req->enables = 0;
5710         if (rmem->nr_pages > 1) {
5711                 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5712                 /* Page size is in log2 units */
5713                 req->page_size = BNXT_PAGE_SHIFT;
5714                 req->page_tbl_depth = 1;
5715         } else {
5716                 req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5717         }
5718         req->fbo = 0;
5719         /* Association of ring index with doorbell index and MSIX number */
5720         req->logical_id = cpu_to_le16(map_index);
5721
5722         switch (ring_type) {
5723         case HWRM_RING_ALLOC_TX: {
5724                 struct bnxt_tx_ring_info *txr;
5725
5726                 txr = container_of(ring, struct bnxt_tx_ring_info,
5727                                    tx_ring_struct);
5728                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5729                 /* Association of transmit ring with completion ring */
5730                 grp_info = &bp->grp_info[ring->grp_idx];
5731                 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5732                 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
5733                 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5734                 req->queue_id = cpu_to_le16(ring->queue_id);
5735                 break;
5736         }
5737         case HWRM_RING_ALLOC_RX:
5738                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5739                 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
5740                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5741                         u16 flags = 0;
5742
5743                         /* Association of rx ring with stats context */
5744                         grp_info = &bp->grp_info[ring->grp_idx];
5745                         req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5746                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5747                         req->enables |= cpu_to_le32(
5748                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5749                         if (NET_IP_ALIGN == 2)
5750                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5751                         req->flags = cpu_to_le16(flags);
5752                 }
5753                 break;
5754         case HWRM_RING_ALLOC_AGG:
5755                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5756                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5757                         /* Association of agg ring with rx ring */
5758                         grp_info = &bp->grp_info[ring->grp_idx];
5759                         req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5760                         req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5761                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5762                         req->enables |= cpu_to_le32(
5763                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5764                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5765                 } else {
5766                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5767                 }
5768                 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5769                 break;
5770         case HWRM_RING_ALLOC_CMPL:
5771                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5772                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5773                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5774                         /* Association of cp ring with nq */
5775                         grp_info = &bp->grp_info[map_index];
5776                         req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5777                         req->cq_handle = cpu_to_le64(ring->handle);
5778                         req->enables |= cpu_to_le32(
5779                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5780                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5781                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5782                 }
5783                 break;
5784         case HWRM_RING_ALLOC_NQ:
5785                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5786                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5787                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5788                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5789                 break;
5790         default:
5791                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5792                            ring_type);
5793                 return -1;
5794         }
5795
5796         resp = hwrm_req_hold(bp, req);
5797         rc = hwrm_req_send(bp, req);
5798         err = le16_to_cpu(resp->error_code);
5799         ring_id = le16_to_cpu(resp->ring_id);
5800         hwrm_req_drop(bp, req);
5801
5802 exit:
5803         if (rc || err) {
5804                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5805                            ring_type, rc, err);
5806                 return -EIO;
5807         }
5808         ring->fw_ring_id = ring_id;
5809         return rc;
5810 }
5811
5812 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5813 {
5814         int rc;
5815
5816         if (BNXT_PF(bp)) {
5817                 struct hwrm_func_cfg_input *req;
5818
5819                 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
5820                 if (rc)
5821                         return rc;
5822
5823                 req->fid = cpu_to_le16(0xffff);
5824                 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5825                 req->async_event_cr = cpu_to_le16(idx);
5826                 return hwrm_req_send(bp, req);
5827         } else {
5828                 struct hwrm_func_vf_cfg_input *req;
5829
5830                 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
5831                 if (rc)
5832                         return rc;
5833
5834                 req->enables =
5835                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5836                 req->async_event_cr = cpu_to_le16(idx);
5837                 return hwrm_req_send(bp, req);
5838         }
5839 }
5840
5841 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5842                         u32 map_idx, u32 xid)
5843 {
5844         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5845                 if (BNXT_PF(bp))
5846                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5847                 else
5848                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5849                 switch (ring_type) {
5850                 case HWRM_RING_ALLOC_TX:
5851                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5852                         break;
5853                 case HWRM_RING_ALLOC_RX:
5854                 case HWRM_RING_ALLOC_AGG:
5855                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5856                         break;
5857                 case HWRM_RING_ALLOC_CMPL:
5858                         db->db_key64 = DBR_PATH_L2;
5859                         break;
5860                 case HWRM_RING_ALLOC_NQ:
5861                         db->db_key64 = DBR_PATH_L2;
5862                         break;
5863                 }
5864                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5865         } else {
5866                 db->doorbell = bp->bar1 + map_idx * 0x80;
5867                 switch (ring_type) {
5868                 case HWRM_RING_ALLOC_TX:
5869                         db->db_key32 = DB_KEY_TX;
5870                         break;
5871                 case HWRM_RING_ALLOC_RX:
5872                 case HWRM_RING_ALLOC_AGG:
5873                         db->db_key32 = DB_KEY_RX;
5874                         break;
5875                 case HWRM_RING_ALLOC_CMPL:
5876                         db->db_key32 = DB_KEY_CP;
5877                         break;
5878                 }
5879         }
5880 }
5881
5882 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5883 {
5884         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5885         int i, rc = 0;
5886         u32 type;
5887
5888         if (bp->flags & BNXT_FLAG_CHIP_P5)
5889                 type = HWRM_RING_ALLOC_NQ;
5890         else
5891                 type = HWRM_RING_ALLOC_CMPL;
5892         for (i = 0; i < bp->cp_nr_rings; i++) {
5893                 struct bnxt_napi *bnapi = bp->bnapi[i];
5894                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5895                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5896                 u32 map_idx = ring->map_idx;
5897                 unsigned int vector;
5898
5899                 vector = bp->irq_tbl[map_idx].vector;
5900                 disable_irq_nosync(vector);
5901                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5902                 if (rc) {
5903                         enable_irq(vector);
5904                         goto err_out;
5905                 }
5906                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5907                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5908                 enable_irq(vector);
5909                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5910
5911                 if (!i) {
5912                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5913                         if (rc)
5914                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5915                 }
5916         }
5917
5918         type = HWRM_RING_ALLOC_TX;
5919         for (i = 0; i < bp->tx_nr_rings; i++) {
5920                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5921                 struct bnxt_ring_struct *ring;
5922                 u32 map_idx;
5923
5924                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5925                         struct bnxt_napi *bnapi = txr->bnapi;
5926                         struct bnxt_cp_ring_info *cpr, *cpr2;
5927                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5928
5929                         cpr = &bnapi->cp_ring;
5930                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5931                         ring = &cpr2->cp_ring_struct;
5932                         ring->handle = BNXT_TX_HDL;
5933                         map_idx = bnapi->index;
5934                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5935                         if (rc)
5936                                 goto err_out;
5937                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5938                                     ring->fw_ring_id);
5939                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5940                 }
5941                 ring = &txr->tx_ring_struct;
5942                 map_idx = i;
5943                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5944                 if (rc)
5945                         goto err_out;
5946                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5947         }
5948
5949         type = HWRM_RING_ALLOC_RX;
5950         for (i = 0; i < bp->rx_nr_rings; i++) {
5951                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5952                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5953                 struct bnxt_napi *bnapi = rxr->bnapi;
5954                 u32 map_idx = bnapi->index;
5955
5956                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5957                 if (rc)
5958                         goto err_out;
5959                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5960                 /* If we have agg rings, post agg buffers first. */
5961                 if (!agg_rings)
5962                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5963                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5964                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5965                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5966                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5967                         struct bnxt_cp_ring_info *cpr2;
5968
5969                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5970                         ring = &cpr2->cp_ring_struct;
5971                         ring->handle = BNXT_RX_HDL;
5972                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5973                         if (rc)
5974                                 goto err_out;
5975                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5976                                     ring->fw_ring_id);
5977                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5978                 }
5979         }
5980
5981         if (agg_rings) {
5982                 type = HWRM_RING_ALLOC_AGG;
5983                 for (i = 0; i < bp->rx_nr_rings; i++) {
5984                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5985                         struct bnxt_ring_struct *ring =
5986                                                 &rxr->rx_agg_ring_struct;
5987                         u32 grp_idx = ring->grp_idx;
5988                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5989
5990                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5991                         if (rc)
5992                                 goto err_out;
5993
5994                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5995                                     ring->fw_ring_id);
5996                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5997                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5998                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5999                 }
6000         }
6001 err_out:
6002         return rc;
6003 }
6004
6005 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6006                                    struct bnxt_ring_struct *ring,
6007                                    u32 ring_type, int cmpl_ring_id)
6008 {
6009         struct hwrm_ring_free_output *resp;
6010         struct hwrm_ring_free_input *req;
6011         u16 error_code = 0;
6012         int rc;
6013
6014         if (BNXT_NO_FW_ACCESS(bp))
6015                 return 0;
6016
6017         rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6018         if (rc)
6019                 goto exit;
6020
6021         req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6022         req->ring_type = ring_type;
6023         req->ring_id = cpu_to_le16(ring->fw_ring_id);
6024
6025         resp = hwrm_req_hold(bp, req);
6026         rc = hwrm_req_send(bp, req);
6027         error_code = le16_to_cpu(resp->error_code);
6028         hwrm_req_drop(bp, req);
6029 exit:
6030         if (rc || error_code) {
6031                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6032                            ring_type, rc, error_code);
6033                 return -EIO;
6034         }
6035         return 0;
6036 }
6037
6038 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6039 {
6040         u32 type;
6041         int i;
6042
6043         if (!bp->bnapi)
6044                 return;
6045
6046         for (i = 0; i < bp->tx_nr_rings; i++) {
6047                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6048                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6049
6050                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6051                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6052
6053                         hwrm_ring_free_send_msg(bp, ring,
6054                                                 RING_FREE_REQ_RING_TYPE_TX,
6055                                                 close_path ? cmpl_ring_id :
6056                                                 INVALID_HW_RING_ID);
6057                         ring->fw_ring_id = INVALID_HW_RING_ID;
6058                 }
6059         }
6060
6061         for (i = 0; i < bp->rx_nr_rings; i++) {
6062                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6063                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6064                 u32 grp_idx = rxr->bnapi->index;
6065
6066                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6067                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6068
6069                         hwrm_ring_free_send_msg(bp, ring,
6070                                                 RING_FREE_REQ_RING_TYPE_RX,
6071                                                 close_path ? cmpl_ring_id :
6072                                                 INVALID_HW_RING_ID);
6073                         ring->fw_ring_id = INVALID_HW_RING_ID;
6074                         bp->grp_info[grp_idx].rx_fw_ring_id =
6075                                 INVALID_HW_RING_ID;
6076                 }
6077         }
6078
6079         if (bp->flags & BNXT_FLAG_CHIP_P5)
6080                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6081         else
6082                 type = RING_FREE_REQ_RING_TYPE_RX;
6083         for (i = 0; i < bp->rx_nr_rings; i++) {
6084                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6085                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6086                 u32 grp_idx = rxr->bnapi->index;
6087
6088                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6089                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6090
6091                         hwrm_ring_free_send_msg(bp, ring, type,
6092                                                 close_path ? cmpl_ring_id :
6093                                                 INVALID_HW_RING_ID);
6094                         ring->fw_ring_id = INVALID_HW_RING_ID;
6095                         bp->grp_info[grp_idx].agg_fw_ring_id =
6096                                 INVALID_HW_RING_ID;
6097                 }
6098         }
6099
6100         /* The completion rings are about to be freed.  After that the
6101          * IRQ doorbell will not work anymore.  So we need to disable
6102          * IRQ here.
6103          */
6104         bnxt_disable_int_sync(bp);
6105
6106         if (bp->flags & BNXT_FLAG_CHIP_P5)
6107                 type = RING_FREE_REQ_RING_TYPE_NQ;
6108         else
6109                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6110         for (i = 0; i < bp->cp_nr_rings; i++) {
6111                 struct bnxt_napi *bnapi = bp->bnapi[i];
6112                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6113                 struct bnxt_ring_struct *ring;
6114                 int j;
6115
6116                 for (j = 0; j < 2; j++) {
6117                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
6118
6119                         if (cpr2) {
6120                                 ring = &cpr2->cp_ring_struct;
6121                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
6122                                         continue;
6123                                 hwrm_ring_free_send_msg(bp, ring,
6124                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
6125                                         INVALID_HW_RING_ID);
6126                                 ring->fw_ring_id = INVALID_HW_RING_ID;
6127                         }
6128                 }
6129                 ring = &cpr->cp_ring_struct;
6130                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6131                         hwrm_ring_free_send_msg(bp, ring, type,
6132                                                 INVALID_HW_RING_ID);
6133                         ring->fw_ring_id = INVALID_HW_RING_ID;
6134                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6135                 }
6136         }
6137 }
6138
6139 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6140                            bool shared);
6141
6142 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6143 {
6144         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6145         struct hwrm_func_qcfg_output *resp;
6146         struct hwrm_func_qcfg_input *req;
6147         int rc;
6148
6149         if (bp->hwrm_spec_code < 0x10601)
6150                 return 0;
6151
6152         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6153         if (rc)
6154                 return rc;
6155
6156         req->fid = cpu_to_le16(0xffff);
6157         resp = hwrm_req_hold(bp, req);
6158         rc = hwrm_req_send(bp, req);
6159         if (rc) {
6160                 hwrm_req_drop(bp, req);
6161                 return rc;
6162         }
6163
6164         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6165         if (BNXT_NEW_RM(bp)) {
6166                 u16 cp, stats;
6167
6168                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6169                 hw_resc->resv_hw_ring_grps =
6170                         le32_to_cpu(resp->alloc_hw_ring_grps);
6171                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6172                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6173                 stats = le16_to_cpu(resp->alloc_stat_ctx);
6174                 hw_resc->resv_irqs = cp;
6175                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6176                         int rx = hw_resc->resv_rx_rings;
6177                         int tx = hw_resc->resv_tx_rings;
6178
6179                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6180                                 rx >>= 1;
6181                         if (cp < (rx + tx)) {
6182                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
6183                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6184                                         rx <<= 1;
6185                                 hw_resc->resv_rx_rings = rx;
6186                                 hw_resc->resv_tx_rings = tx;
6187                         }
6188                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6189                         hw_resc->resv_hw_ring_grps = rx;
6190                 }
6191                 hw_resc->resv_cp_rings = cp;
6192                 hw_resc->resv_stat_ctxs = stats;
6193         }
6194         hwrm_req_drop(bp, req);
6195         return 0;
6196 }
6197
6198 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6199 {
6200         struct hwrm_func_qcfg_output *resp;
6201         struct hwrm_func_qcfg_input *req;
6202         int rc;
6203
6204         if (bp->hwrm_spec_code < 0x10601)
6205                 return 0;
6206
6207         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6208         if (rc)
6209                 return rc;
6210
6211         req->fid = cpu_to_le16(fid);
6212         resp = hwrm_req_hold(bp, req);
6213         rc = hwrm_req_send(bp, req);
6214         if (!rc)
6215                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6216
6217         hwrm_req_drop(bp, req);
6218         return rc;
6219 }
6220
6221 static bool bnxt_rfs_supported(struct bnxt *bp);
6222
6223 static struct hwrm_func_cfg_input *
6224 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6225                              int ring_grps, int cp_rings, int stats, int vnics)
6226 {
6227         struct hwrm_func_cfg_input *req;
6228         u32 enables = 0;
6229
6230         if (hwrm_req_init(bp, req, HWRM_FUNC_CFG))
6231                 return NULL;
6232
6233         req->fid = cpu_to_le16(0xffff);
6234         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6235         req->num_tx_rings = cpu_to_le16(tx_rings);
6236         if (BNXT_NEW_RM(bp)) {
6237                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6238                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6239                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6240                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6241                         enables |= tx_rings + ring_grps ?
6242                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6243                         enables |= rx_rings ?
6244                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6245                 } else {
6246                         enables |= cp_rings ?
6247                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6248                         enables |= ring_grps ?
6249                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6250                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6251                 }
6252                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6253
6254                 req->num_rx_rings = cpu_to_le16(rx_rings);
6255                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6256                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6257                         req->num_msix = cpu_to_le16(cp_rings);
6258                         req->num_rsscos_ctxs =
6259                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6260                 } else {
6261                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6262                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6263                         req->num_rsscos_ctxs = cpu_to_le16(1);
6264                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6265                             bnxt_rfs_supported(bp))
6266                                 req->num_rsscos_ctxs =
6267                                         cpu_to_le16(ring_grps + 1);
6268                 }
6269                 req->num_stat_ctxs = cpu_to_le16(stats);
6270                 req->num_vnics = cpu_to_le16(vnics);
6271         }
6272         req->enables = cpu_to_le32(enables);
6273         return req;
6274 }
6275
6276 static struct hwrm_func_vf_cfg_input *
6277 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6278                              int ring_grps, int cp_rings, int stats, int vnics)
6279 {
6280         struct hwrm_func_vf_cfg_input *req;
6281         u32 enables = 0;
6282
6283         if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
6284                 return NULL;
6285
6286         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6287         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6288                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6289         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6290         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6291                 enables |= tx_rings + ring_grps ?
6292                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6293         } else {
6294                 enables |= cp_rings ?
6295                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6296                 enables |= ring_grps ?
6297                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6298         }
6299         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6300         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6301
6302         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6303         req->num_tx_rings = cpu_to_le16(tx_rings);
6304         req->num_rx_rings = cpu_to_le16(rx_rings);
6305         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6306                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6307                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6308         } else {
6309                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6310                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6311                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6312         }
6313         req->num_stat_ctxs = cpu_to_le16(stats);
6314         req->num_vnics = cpu_to_le16(vnics);
6315
6316         req->enables = cpu_to_le32(enables);
6317         return req;
6318 }
6319
6320 static int
6321 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6322                            int ring_grps, int cp_rings, int stats, int vnics)
6323 {
6324         struct hwrm_func_cfg_input *req;
6325         int rc;
6326
6327         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6328                                            cp_rings, stats, vnics);
6329         if (!req)
6330                 return -ENOMEM;
6331
6332         if (!req->enables) {
6333                 hwrm_req_drop(bp, req);
6334                 return 0;
6335         }
6336
6337         rc = hwrm_req_send(bp, req);
6338         if (rc)
6339                 return rc;
6340
6341         if (bp->hwrm_spec_code < 0x10601)
6342                 bp->hw_resc.resv_tx_rings = tx_rings;
6343
6344         return bnxt_hwrm_get_rings(bp);
6345 }
6346
6347 static int
6348 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6349                            int ring_grps, int cp_rings, int stats, int vnics)
6350 {
6351         struct hwrm_func_vf_cfg_input *req;
6352         int rc;
6353
6354         if (!BNXT_NEW_RM(bp)) {
6355                 bp->hw_resc.resv_tx_rings = tx_rings;
6356                 return 0;
6357         }
6358
6359         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6360                                            cp_rings, stats, vnics);
6361         if (!req)
6362                 return -ENOMEM;
6363
6364         rc = hwrm_req_send(bp, req);
6365         if (rc)
6366                 return rc;
6367
6368         return bnxt_hwrm_get_rings(bp);
6369 }
6370
6371 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6372                                    int cp, int stat, int vnic)
6373 {
6374         if (BNXT_PF(bp))
6375                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6376                                                   vnic);
6377         else
6378                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6379                                                   vnic);
6380 }
6381
6382 int bnxt_nq_rings_in_use(struct bnxt *bp)
6383 {
6384         int cp = bp->cp_nr_rings;
6385         int ulp_msix, ulp_base;
6386
6387         ulp_msix = bnxt_get_ulp_msix_num(bp);
6388         if (ulp_msix) {
6389                 ulp_base = bnxt_get_ulp_msix_base(bp);
6390                 cp += ulp_msix;
6391                 if ((ulp_base + ulp_msix) > cp)
6392                         cp = ulp_base + ulp_msix;
6393         }
6394         return cp;
6395 }
6396
6397 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6398 {
6399         int cp;
6400
6401         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6402                 return bnxt_nq_rings_in_use(bp);
6403
6404         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6405         return cp;
6406 }
6407
6408 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6409 {
6410         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6411         int cp = bp->cp_nr_rings;
6412
6413         if (!ulp_stat)
6414                 return cp;
6415
6416         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6417                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6418
6419         return cp + ulp_stat;
6420 }
6421
6422 /* Check if a default RSS map needs to be setup.  This function is only
6423  * used on older firmware that does not require reserving RX rings.
6424  */
6425 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6426 {
6427         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6428
6429         /* The RSS map is valid for RX rings set to resv_rx_rings */
6430         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6431                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6432                 if (!netif_is_rxfh_configured(bp->dev))
6433                         bnxt_set_dflt_rss_indir_tbl(bp);
6434         }
6435 }
6436
6437 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6438 {
6439         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6440         int cp = bnxt_cp_rings_in_use(bp);
6441         int nq = bnxt_nq_rings_in_use(bp);
6442         int rx = bp->rx_nr_rings, stat;
6443         int vnic = 1, grp = rx;
6444
6445         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6446             bp->hwrm_spec_code >= 0x10601)
6447                 return true;
6448
6449         /* Old firmware does not need RX ring reservations but we still
6450          * need to setup a default RSS map when needed.  With new firmware
6451          * we go through RX ring reservations first and then set up the
6452          * RSS map for the successfully reserved RX rings when needed.
6453          */
6454         if (!BNXT_NEW_RM(bp)) {
6455                 bnxt_check_rss_tbl_no_rmgr(bp);
6456                 return false;
6457         }
6458         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6459                 vnic = rx + 1;
6460         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6461                 rx <<= 1;
6462         stat = bnxt_get_func_stat_ctxs(bp);
6463         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6464             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6465             (hw_resc->resv_hw_ring_grps != grp &&
6466              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6467                 return true;
6468         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6469             hw_resc->resv_irqs != nq)
6470                 return true;
6471         return false;
6472 }
6473
6474 static int __bnxt_reserve_rings(struct bnxt *bp)
6475 {
6476         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6477         int cp = bnxt_nq_rings_in_use(bp);
6478         int tx = bp->tx_nr_rings;
6479         int rx = bp->rx_nr_rings;
6480         int grp, rx_rings, rc;
6481         int vnic = 1, stat;
6482         bool sh = false;
6483
6484         if (!bnxt_need_reserve_rings(bp))
6485                 return 0;
6486
6487         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6488                 sh = true;
6489         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6490                 vnic = rx + 1;
6491         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6492                 rx <<= 1;
6493         grp = bp->rx_nr_rings;
6494         stat = bnxt_get_func_stat_ctxs(bp);
6495
6496         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6497         if (rc)
6498                 return rc;
6499
6500         tx = hw_resc->resv_tx_rings;
6501         if (BNXT_NEW_RM(bp)) {
6502                 rx = hw_resc->resv_rx_rings;
6503                 cp = hw_resc->resv_irqs;
6504                 grp = hw_resc->resv_hw_ring_grps;
6505                 vnic = hw_resc->resv_vnics;
6506                 stat = hw_resc->resv_stat_ctxs;
6507         }
6508
6509         rx_rings = rx;
6510         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6511                 if (rx >= 2) {
6512                         rx_rings = rx >> 1;
6513                 } else {
6514                         if (netif_running(bp->dev))
6515                                 return -ENOMEM;
6516
6517                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6518                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6519                         bp->dev->hw_features &= ~NETIF_F_LRO;
6520                         bp->dev->features &= ~NETIF_F_LRO;
6521                         bnxt_set_ring_params(bp);
6522                 }
6523         }
6524         rx_rings = min_t(int, rx_rings, grp);
6525         cp = min_t(int, cp, bp->cp_nr_rings);
6526         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6527                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6528         cp = min_t(int, cp, stat);
6529         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6530         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6531                 rx = rx_rings << 1;
6532         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6533         bp->tx_nr_rings = tx;
6534
6535         /* If we cannot reserve all the RX rings, reset the RSS map only
6536          * if absolutely necessary
6537          */
6538         if (rx_rings != bp->rx_nr_rings) {
6539                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6540                             rx_rings, bp->rx_nr_rings);
6541                 if (netif_is_rxfh_configured(bp->dev) &&
6542                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6543                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6544                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6545                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6546                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6547                 }
6548         }
6549         bp->rx_nr_rings = rx_rings;
6550         bp->cp_nr_rings = cp;
6551
6552         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6553                 return -ENOMEM;
6554
6555         if (!netif_is_rxfh_configured(bp->dev))
6556                 bnxt_set_dflt_rss_indir_tbl(bp);
6557
6558         return rc;
6559 }
6560
6561 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6562                                     int ring_grps, int cp_rings, int stats,
6563                                     int vnics)
6564 {
6565         struct hwrm_func_vf_cfg_input *req;
6566         u32 flags;
6567
6568         if (!BNXT_NEW_RM(bp))
6569                 return 0;
6570
6571         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6572                                            cp_rings, stats, vnics);
6573         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6574                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6575                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6576                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6577                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6578                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6579         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6580                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6581
6582         req->flags = cpu_to_le32(flags);
6583         return hwrm_req_send_silent(bp, req);
6584 }
6585
6586 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6587                                     int ring_grps, int cp_rings, int stats,
6588                                     int vnics)
6589 {
6590         struct hwrm_func_cfg_input *req;
6591         u32 flags;
6592
6593         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6594                                            cp_rings, stats, vnics);
6595         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6596         if (BNXT_NEW_RM(bp)) {
6597                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6598                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6599                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6600                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6601                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6602                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6603                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6604                 else
6605                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6606         }
6607
6608         req->flags = cpu_to_le32(flags);
6609         return hwrm_req_send_silent(bp, req);
6610 }
6611
6612 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6613                                  int ring_grps, int cp_rings, int stats,
6614                                  int vnics)
6615 {
6616         if (bp->hwrm_spec_code < 0x10801)
6617                 return 0;
6618
6619         if (BNXT_PF(bp))
6620                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6621                                                 ring_grps, cp_rings, stats,
6622                                                 vnics);
6623
6624         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6625                                         cp_rings, stats, vnics);
6626 }
6627
6628 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6629 {
6630         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6631         struct hwrm_ring_aggint_qcaps_output *resp;
6632         struct hwrm_ring_aggint_qcaps_input *req;
6633         int rc;
6634
6635         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6636         coal_cap->num_cmpl_dma_aggr_max = 63;
6637         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6638         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6639         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6640         coal_cap->int_lat_tmr_min_max = 65535;
6641         coal_cap->int_lat_tmr_max_max = 65535;
6642         coal_cap->num_cmpl_aggr_int_max = 65535;
6643         coal_cap->timer_units = 80;
6644
6645         if (bp->hwrm_spec_code < 0x10902)
6646                 return;
6647
6648         if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
6649                 return;
6650
6651         resp = hwrm_req_hold(bp, req);
6652         rc = hwrm_req_send_silent(bp, req);
6653         if (!rc) {
6654                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6655                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6656                 coal_cap->num_cmpl_dma_aggr_max =
6657                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6658                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6659                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6660                 coal_cap->cmpl_aggr_dma_tmr_max =
6661                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6662                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6663                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6664                 coal_cap->int_lat_tmr_min_max =
6665                         le16_to_cpu(resp->int_lat_tmr_min_max);
6666                 coal_cap->int_lat_tmr_max_max =
6667                         le16_to_cpu(resp->int_lat_tmr_max_max);
6668                 coal_cap->num_cmpl_aggr_int_max =
6669                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6670                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6671         }
6672         hwrm_req_drop(bp, req);
6673 }
6674
6675 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6676 {
6677         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6678
6679         return usec * 1000 / coal_cap->timer_units;
6680 }
6681
6682 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6683         struct bnxt_coal *hw_coal,
6684         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6685 {
6686         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6687         u16 val, tmr, max, flags = hw_coal->flags;
6688         u32 cmpl_params = coal_cap->cmpl_params;
6689
6690         max = hw_coal->bufs_per_record * 128;
6691         if (hw_coal->budget)
6692                 max = hw_coal->bufs_per_record * hw_coal->budget;
6693         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6694
6695         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6696         req->num_cmpl_aggr_int = cpu_to_le16(val);
6697
6698         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6699         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6700
6701         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6702                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6703         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6704
6705         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6706         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6707         req->int_lat_tmr_max = cpu_to_le16(tmr);
6708
6709         /* min timer set to 1/2 of interrupt timer */
6710         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6711                 val = tmr / 2;
6712                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6713                 req->int_lat_tmr_min = cpu_to_le16(val);
6714                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6715         }
6716
6717         /* buf timer set to 1/4 of interrupt timer */
6718         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6719         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6720
6721         if (cmpl_params &
6722             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6723                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6724                 val = clamp_t(u16, tmr, 1,
6725                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6726                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6727                 req->enables |=
6728                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6729         }
6730
6731         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6732             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6733                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6734         req->flags = cpu_to_le16(flags);
6735         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6736 }
6737
6738 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6739                                    struct bnxt_coal *hw_coal)
6740 {
6741         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
6742         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6743         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6744         u32 nq_params = coal_cap->nq_params;
6745         u16 tmr;
6746         int rc;
6747
6748         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6749                 return 0;
6750
6751         rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6752         if (rc)
6753                 return rc;
6754
6755         req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6756         req->flags =
6757                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6758
6759         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6760         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6761         req->int_lat_tmr_min = cpu_to_le16(tmr);
6762         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6763         return hwrm_req_send(bp, req);
6764 }
6765
6766 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6767 {
6768         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
6769         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6770         struct bnxt_coal coal;
6771         int rc;
6772
6773         /* Tick values in micro seconds.
6774          * 1 coal_buf x bufs_per_record = 1 completion record.
6775          */
6776         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6777
6778         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6779         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6780
6781         if (!bnapi->rx_ring)
6782                 return -ENODEV;
6783
6784         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6785         if (rc)
6786                 return rc;
6787
6788         bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
6789
6790         req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6791
6792         return hwrm_req_send(bp, req_rx);
6793 }
6794
6795 int bnxt_hwrm_set_coal(struct bnxt *bp)
6796 {
6797         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
6798                                                            *req;
6799         int i, rc;
6800
6801         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6802         if (rc)
6803                 return rc;
6804
6805         rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6806         if (rc) {
6807                 hwrm_req_drop(bp, req_rx);
6808                 return rc;
6809         }
6810
6811         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
6812         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
6813
6814         hwrm_req_hold(bp, req_rx);
6815         hwrm_req_hold(bp, req_tx);
6816         for (i = 0; i < bp->cp_nr_rings; i++) {
6817                 struct bnxt_napi *bnapi = bp->bnapi[i];
6818                 struct bnxt_coal *hw_coal;
6819                 u16 ring_id;
6820
6821                 req = req_rx;
6822                 if (!bnapi->rx_ring) {
6823                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6824                         req = req_tx;
6825                 } else {
6826                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6827                 }
6828                 req->ring_id = cpu_to_le16(ring_id);
6829
6830                 rc = hwrm_req_send(bp, req);
6831                 if (rc)
6832                         break;
6833
6834                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6835                         continue;
6836
6837                 if (bnapi->rx_ring && bnapi->tx_ring) {
6838                         req = req_tx;
6839                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6840                         req->ring_id = cpu_to_le16(ring_id);
6841                         rc = hwrm_req_send(bp, req);
6842                         if (rc)
6843                                 break;
6844                 }
6845                 if (bnapi->rx_ring)
6846                         hw_coal = &bp->rx_coal;
6847                 else
6848                         hw_coal = &bp->tx_coal;
6849                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6850         }
6851         hwrm_req_drop(bp, req_rx);
6852         hwrm_req_drop(bp, req_tx);
6853         return rc;
6854 }
6855
6856 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6857 {
6858         struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
6859         struct hwrm_stat_ctx_free_input *req;
6860         int i;
6861
6862         if (!bp->bnapi)
6863                 return;
6864
6865         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6866                 return;
6867
6868         if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
6869                 return;
6870         if (BNXT_FW_MAJ(bp) <= 20) {
6871                 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
6872                         hwrm_req_drop(bp, req);
6873                         return;
6874                 }
6875                 hwrm_req_hold(bp, req0);
6876         }
6877         hwrm_req_hold(bp, req);
6878         for (i = 0; i < bp->cp_nr_rings; i++) {
6879                 struct bnxt_napi *bnapi = bp->bnapi[i];
6880                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6881
6882                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6883                         req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6884                         if (req0) {
6885                                 req0->stat_ctx_id = req->stat_ctx_id;
6886                                 hwrm_req_send(bp, req0);
6887                         }
6888                         hwrm_req_send(bp, req);
6889
6890                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6891                 }
6892         }
6893         hwrm_req_drop(bp, req);
6894         if (req0)
6895                 hwrm_req_drop(bp, req0);
6896 }
6897
6898 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6899 {
6900         struct hwrm_stat_ctx_alloc_output *resp;
6901         struct hwrm_stat_ctx_alloc_input *req;
6902         int rc, i;
6903
6904         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6905                 return 0;
6906
6907         rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
6908         if (rc)
6909                 return rc;
6910
6911         req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6912         req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6913
6914         resp = hwrm_req_hold(bp, req);
6915         for (i = 0; i < bp->cp_nr_rings; i++) {
6916                 struct bnxt_napi *bnapi = bp->bnapi[i];
6917                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6918
6919                 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6920
6921                 rc = hwrm_req_send(bp, req);
6922                 if (rc)
6923                         break;
6924
6925                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6926
6927                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6928         }
6929         hwrm_req_drop(bp, req);
6930         return rc;
6931 }
6932
6933 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6934 {
6935         struct hwrm_func_qcfg_output *resp;
6936         struct hwrm_func_qcfg_input *req;
6937         u32 min_db_offset = 0;
6938         u16 flags;
6939         int rc;
6940
6941         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6942         if (rc)
6943                 return rc;
6944
6945         req->fid = cpu_to_le16(0xffff);
6946         resp = hwrm_req_hold(bp, req);
6947         rc = hwrm_req_send(bp, req);
6948         if (rc)
6949                 goto func_qcfg_exit;
6950
6951 #ifdef CONFIG_BNXT_SRIOV
6952         if (BNXT_VF(bp)) {
6953                 struct bnxt_vf_info *vf = &bp->vf;
6954
6955                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6956         } else {
6957                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6958         }
6959 #endif
6960         flags = le16_to_cpu(resp->flags);
6961         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6962                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6963                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6964                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6965                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6966         }
6967         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6968                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6969
6970         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6971                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6972
6973         switch (resp->port_partition_type) {
6974         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6975         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6976         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6977                 bp->port_partition_type = resp->port_partition_type;
6978                 break;
6979         }
6980         if (bp->hwrm_spec_code < 0x10707 ||
6981             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6982                 bp->br_mode = BRIDGE_MODE_VEB;
6983         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6984                 bp->br_mode = BRIDGE_MODE_VEPA;
6985         else
6986                 bp->br_mode = BRIDGE_MODE_UNDEF;
6987
6988         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6989         if (!bp->max_mtu)
6990                 bp->max_mtu = BNXT_MAX_MTU;
6991
6992         if (bp->db_size)
6993                 goto func_qcfg_exit;
6994
6995         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6996                 if (BNXT_PF(bp))
6997                         min_db_offset = DB_PF_OFFSET_P5;
6998                 else
6999                         min_db_offset = DB_VF_OFFSET_P5;
7000         }
7001         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7002                                  1024);
7003         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7004             bp->db_size <= min_db_offset)
7005                 bp->db_size = pci_resource_len(bp->pdev, 2);
7006
7007 func_qcfg_exit:
7008         hwrm_req_drop(bp, req);
7009         return rc;
7010 }
7011
7012 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
7013                         struct hwrm_func_backing_store_qcaps_output *resp)
7014 {
7015         struct bnxt_mem_init *mem_init;
7016         u16 init_mask;
7017         u8 init_val;
7018         u8 *offset;
7019         int i;
7020
7021         init_val = resp->ctx_kind_initializer;
7022         init_mask = le16_to_cpu(resp->ctx_init_mask);
7023         offset = &resp->qp_init_offset;
7024         mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7025         for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
7026                 mem_init->init_val = init_val;
7027                 mem_init->offset = BNXT_MEM_INVALID_OFFSET;
7028                 if (!init_mask)
7029                         continue;
7030                 if (i == BNXT_CTX_MEM_INIT_STAT)
7031                         offset = &resp->stat_init_offset;
7032                 if (init_mask & (1 << i))
7033                         mem_init->offset = *offset * 4;
7034                 else
7035                         mem_init->init_val = 0;
7036         }
7037         ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
7038         ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
7039         ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
7040         ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
7041         ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
7042         ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
7043 }
7044
7045 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7046 {
7047         struct hwrm_func_backing_store_qcaps_output *resp;
7048         struct hwrm_func_backing_store_qcaps_input *req;
7049         int rc;
7050
7051         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7052                 return 0;
7053
7054         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7055         if (rc)
7056                 return rc;
7057
7058         resp = hwrm_req_hold(bp, req);
7059         rc = hwrm_req_send_silent(bp, req);
7060         if (!rc) {
7061                 struct bnxt_ctx_pg_info *ctx_pg;
7062                 struct bnxt_ctx_mem_info *ctx;
7063                 int i, tqm_rings;
7064
7065                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7066                 if (!ctx) {
7067                         rc = -ENOMEM;
7068                         goto ctx_err;
7069                 }
7070                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
7071                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7072                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7073                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
7074                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7075                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
7076                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
7077                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7078                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
7079                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
7080                 ctx->vnic_max_vnic_entries =
7081                         le16_to_cpu(resp->vnic_max_vnic_entries);
7082                 ctx->vnic_max_ring_table_entries =
7083                         le16_to_cpu(resp->vnic_max_ring_table_entries);
7084                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
7085                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
7086                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
7087                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
7088                 ctx->tqm_min_entries_per_ring =
7089                         le32_to_cpu(resp->tqm_min_entries_per_ring);
7090                 ctx->tqm_max_entries_per_ring =
7091                         le32_to_cpu(resp->tqm_max_entries_per_ring);
7092                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
7093                 if (!ctx->tqm_entries_multiple)
7094                         ctx->tqm_entries_multiple = 1;
7095                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
7096                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
7097                 ctx->mrav_num_entries_units =
7098                         le16_to_cpu(resp->mrav_num_entries_units);
7099                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
7100                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
7101
7102                 bnxt_init_ctx_initializer(ctx, resp);
7103
7104                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7105                 if (!ctx->tqm_fp_rings_count)
7106                         ctx->tqm_fp_rings_count = bp->max_q;
7107                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7108                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7109
7110                 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
7111                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
7112                 if (!ctx_pg) {
7113                         kfree(ctx);
7114                         rc = -ENOMEM;
7115                         goto ctx_err;
7116                 }
7117                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
7118                         ctx->tqm_mem[i] = ctx_pg;
7119                 bp->ctx = ctx;
7120         } else {
7121                 rc = 0;
7122         }
7123 ctx_err:
7124         hwrm_req_drop(bp, req);
7125         return rc;
7126 }
7127
7128 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
7129                                   __le64 *pg_dir)
7130 {
7131         if (!rmem->nr_pages)
7132                 return;
7133
7134         BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
7135         if (rmem->depth >= 1) {
7136                 if (rmem->depth == 2)
7137                         *pg_attr |= 2;
7138                 else
7139                         *pg_attr |= 1;
7140                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
7141         } else {
7142                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
7143         }
7144 }
7145
7146 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
7147         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
7148          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
7149          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
7150          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
7151          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
7152
7153 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
7154 {
7155         struct hwrm_func_backing_store_cfg_input *req;
7156         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7157         struct bnxt_ctx_pg_info *ctx_pg;
7158         void **__req = (void **)&req;
7159         u32 req_len = sizeof(*req);
7160         __le32 *num_entries;
7161         __le64 *pg_dir;
7162         u32 flags = 0;
7163         u8 *pg_attr;
7164         u32 ena;
7165         int rc;
7166         int i;
7167
7168         if (!ctx)
7169                 return 0;
7170
7171         if (req_len > bp->hwrm_max_ext_req_len)
7172                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
7173         rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
7174         if (rc)
7175                 return rc;
7176
7177         req->enables = cpu_to_le32(enables);
7178         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
7179                 ctx_pg = &ctx->qp_mem;
7180                 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
7181                 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
7182                 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
7183                 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
7184                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7185                                       &req->qpc_pg_size_qpc_lvl,
7186                                       &req->qpc_page_dir);
7187         }
7188         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
7189                 ctx_pg = &ctx->srq_mem;
7190                 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
7191                 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
7192                 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
7193                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7194                                       &req->srq_pg_size_srq_lvl,
7195                                       &req->srq_page_dir);
7196         }
7197         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7198                 ctx_pg = &ctx->cq_mem;
7199                 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
7200                 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7201                 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7202                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7203                                       &req->cq_pg_size_cq_lvl,
7204                                       &req->cq_page_dir);
7205         }
7206         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7207                 ctx_pg = &ctx->vnic_mem;
7208                 req->vnic_num_vnic_entries =
7209                         cpu_to_le16(ctx->vnic_max_vnic_entries);
7210                 req->vnic_num_ring_table_entries =
7211                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
7212                 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7213                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7214                                       &req->vnic_pg_size_vnic_lvl,
7215                                       &req->vnic_page_dir);
7216         }
7217         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7218                 ctx_pg = &ctx->stat_mem;
7219                 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7220                 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7221                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7222                                       &req->stat_pg_size_stat_lvl,
7223                                       &req->stat_page_dir);
7224         }
7225         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7226                 ctx_pg = &ctx->mrav_mem;
7227                 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7228                 if (ctx->mrav_num_entries_units)
7229                         flags |=
7230                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7231                 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7232                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7233                                       &req->mrav_pg_size_mrav_lvl,
7234                                       &req->mrav_page_dir);
7235         }
7236         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7237                 ctx_pg = &ctx->tim_mem;
7238                 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
7239                 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7240                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7241                                       &req->tim_pg_size_tim_lvl,
7242                                       &req->tim_page_dir);
7243         }
7244         for (i = 0, num_entries = &req->tqm_sp_num_entries,
7245              pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
7246              pg_dir = &req->tqm_sp_page_dir,
7247              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7248              i < BNXT_MAX_TQM_RINGS;
7249              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7250                 if (!(enables & ena))
7251                         continue;
7252
7253                 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7254                 ctx_pg = ctx->tqm_mem[i];
7255                 *num_entries = cpu_to_le32(ctx_pg->entries);
7256                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7257         }
7258         req->flags = cpu_to_le32(flags);
7259         return hwrm_req_send(bp, req);
7260 }
7261
7262 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7263                                   struct bnxt_ctx_pg_info *ctx_pg)
7264 {
7265         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7266
7267         rmem->page_size = BNXT_PAGE_SIZE;
7268         rmem->pg_arr = ctx_pg->ctx_pg_arr;
7269         rmem->dma_arr = ctx_pg->ctx_dma_arr;
7270         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7271         if (rmem->depth >= 1)
7272                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7273         return bnxt_alloc_ring(bp, rmem);
7274 }
7275
7276 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7277                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7278                                   u8 depth, struct bnxt_mem_init *mem_init)
7279 {
7280         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7281         int rc;
7282
7283         if (!mem_size)
7284                 return -EINVAL;
7285
7286         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7287         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7288                 ctx_pg->nr_pages = 0;
7289                 return -EINVAL;
7290         }
7291         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7292                 int nr_tbls, i;
7293
7294                 rmem->depth = 2;
7295                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7296                                              GFP_KERNEL);
7297                 if (!ctx_pg->ctx_pg_tbl)
7298                         return -ENOMEM;
7299                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7300                 rmem->nr_pages = nr_tbls;
7301                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7302                 if (rc)
7303                         return rc;
7304                 for (i = 0; i < nr_tbls; i++) {
7305                         struct bnxt_ctx_pg_info *pg_tbl;
7306
7307                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7308                         if (!pg_tbl)
7309                                 return -ENOMEM;
7310                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7311                         rmem = &pg_tbl->ring_mem;
7312                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7313                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7314                         rmem->depth = 1;
7315                         rmem->nr_pages = MAX_CTX_PAGES;
7316                         rmem->mem_init = mem_init;
7317                         if (i == (nr_tbls - 1)) {
7318                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7319
7320                                 if (rem)
7321                                         rmem->nr_pages = rem;
7322                         }
7323                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7324                         if (rc)
7325                                 break;
7326                 }
7327         } else {
7328                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7329                 if (rmem->nr_pages > 1 || depth)
7330                         rmem->depth = 1;
7331                 rmem->mem_init = mem_init;
7332                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7333         }
7334         return rc;
7335 }
7336
7337 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7338                                   struct bnxt_ctx_pg_info *ctx_pg)
7339 {
7340         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7341
7342         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7343             ctx_pg->ctx_pg_tbl) {
7344                 int i, nr_tbls = rmem->nr_pages;
7345
7346                 for (i = 0; i < nr_tbls; i++) {
7347                         struct bnxt_ctx_pg_info *pg_tbl;
7348                         struct bnxt_ring_mem_info *rmem2;
7349
7350                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7351                         if (!pg_tbl)
7352                                 continue;
7353                         rmem2 = &pg_tbl->ring_mem;
7354                         bnxt_free_ring(bp, rmem2);
7355                         ctx_pg->ctx_pg_arr[i] = NULL;
7356                         kfree(pg_tbl);
7357                         ctx_pg->ctx_pg_tbl[i] = NULL;
7358                 }
7359                 kfree(ctx_pg->ctx_pg_tbl);
7360                 ctx_pg->ctx_pg_tbl = NULL;
7361         }
7362         bnxt_free_ring(bp, rmem);
7363         ctx_pg->nr_pages = 0;
7364 }
7365
7366 void bnxt_free_ctx_mem(struct bnxt *bp)
7367 {
7368         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7369         int i;
7370
7371         if (!ctx)
7372                 return;
7373
7374         if (ctx->tqm_mem[0]) {
7375                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7376                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7377                 kfree(ctx->tqm_mem[0]);
7378                 ctx->tqm_mem[0] = NULL;
7379         }
7380
7381         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7382         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7383         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7384         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7385         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7386         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7387         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7388         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7389 }
7390
7391 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7392 {
7393         struct bnxt_ctx_pg_info *ctx_pg;
7394         struct bnxt_ctx_mem_info *ctx;
7395         struct bnxt_mem_init *init;
7396         u32 mem_size, ena, entries;
7397         u32 entries_sp, min;
7398         u32 num_mr, num_ah;
7399         u32 extra_srqs = 0;
7400         u32 extra_qps = 0;
7401         u8 pg_lvl = 1;
7402         int i, rc;
7403
7404         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7405         if (rc) {
7406                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7407                            rc);
7408                 return rc;
7409         }
7410         ctx = bp->ctx;
7411         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7412                 return 0;
7413
7414         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7415                 pg_lvl = 2;
7416                 extra_qps = 65536;
7417                 extra_srqs = 8192;
7418         }
7419
7420         ctx_pg = &ctx->qp_mem;
7421         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7422                           extra_qps;
7423         if (ctx->qp_entry_size) {
7424                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7425                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7426                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7427                 if (rc)
7428                         return rc;
7429         }
7430
7431         ctx_pg = &ctx->srq_mem;
7432         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7433         if (ctx->srq_entry_size) {
7434                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7435                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7436                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7437                 if (rc)
7438                         return rc;
7439         }
7440
7441         ctx_pg = &ctx->cq_mem;
7442         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7443         if (ctx->cq_entry_size) {
7444                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7445                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7446                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7447                 if (rc)
7448                         return rc;
7449         }
7450
7451         ctx_pg = &ctx->vnic_mem;
7452         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7453                           ctx->vnic_max_ring_table_entries;
7454         if (ctx->vnic_entry_size) {
7455                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7456                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7457                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7458                 if (rc)
7459                         return rc;
7460         }
7461
7462         ctx_pg = &ctx->stat_mem;
7463         ctx_pg->entries = ctx->stat_max_entries;
7464         if (ctx->stat_entry_size) {
7465                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7466                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7467                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7468                 if (rc)
7469                         return rc;
7470         }
7471
7472         ena = 0;
7473         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7474                 goto skip_rdma;
7475
7476         ctx_pg = &ctx->mrav_mem;
7477         /* 128K extra is needed to accommodate static AH context
7478          * allocation by f/w.
7479          */
7480         num_mr = 1024 * 256;
7481         num_ah = 1024 * 128;
7482         ctx_pg->entries = num_mr + num_ah;
7483         if (ctx->mrav_entry_size) {
7484                 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7485                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7486                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7487                 if (rc)
7488                         return rc;
7489         }
7490         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7491         if (ctx->mrav_num_entries_units)
7492                 ctx_pg->entries =
7493                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7494                          (num_ah / ctx->mrav_num_entries_units);
7495
7496         ctx_pg = &ctx->tim_mem;
7497         ctx_pg->entries = ctx->qp_mem.entries;
7498         if (ctx->tim_entry_size) {
7499                 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7500                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7501                 if (rc)
7502                         return rc;
7503         }
7504         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7505
7506 skip_rdma:
7507         min = ctx->tqm_min_entries_per_ring;
7508         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7509                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7510         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7511         entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7512         entries = roundup(entries, ctx->tqm_entries_multiple);
7513         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7514         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7515                 ctx_pg = ctx->tqm_mem[i];
7516                 ctx_pg->entries = i ? entries : entries_sp;
7517                 if (ctx->tqm_entry_size) {
7518                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7519                         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7520                                                     NULL);
7521                         if (rc)
7522                                 return rc;
7523                 }
7524                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7525         }
7526         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7527         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7528         if (rc) {
7529                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7530                            rc);
7531                 return rc;
7532         }
7533         ctx->flags |= BNXT_CTX_FLAG_INITED;
7534         return 0;
7535 }
7536
7537 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7538 {
7539         struct hwrm_func_resource_qcaps_output *resp;
7540         struct hwrm_func_resource_qcaps_input *req;
7541         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7542         int rc;
7543
7544         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
7545         if (rc)
7546                 return rc;
7547
7548         req->fid = cpu_to_le16(0xffff);
7549         resp = hwrm_req_hold(bp, req);
7550         rc = hwrm_req_send_silent(bp, req);
7551         if (rc)
7552                 goto hwrm_func_resc_qcaps_exit;
7553
7554         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7555         if (!all)
7556                 goto hwrm_func_resc_qcaps_exit;
7557
7558         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7559         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7560         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7561         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7562         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7563         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7564         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7565         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7566         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7567         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7568         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7569         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7570         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7571         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7572         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7573         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7574
7575         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7576                 u16 max_msix = le16_to_cpu(resp->max_msix);
7577
7578                 hw_resc->max_nqs = max_msix;
7579                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7580         }
7581
7582         if (BNXT_PF(bp)) {
7583                 struct bnxt_pf_info *pf = &bp->pf;
7584
7585                 pf->vf_resv_strategy =
7586                         le16_to_cpu(resp->vf_reservation_strategy);
7587                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7588                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7589         }
7590 hwrm_func_resc_qcaps_exit:
7591         hwrm_req_drop(bp, req);
7592         return rc;
7593 }
7594
7595 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7596 {
7597         struct hwrm_port_mac_ptp_qcfg_output *resp;
7598         struct hwrm_port_mac_ptp_qcfg_input *req;
7599         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7600         bool phc_cfg;
7601         u8 flags;
7602         int rc;
7603
7604         if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) {
7605                 rc = -ENODEV;
7606                 goto no_ptp;
7607         }
7608
7609         rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
7610         if (rc)
7611                 goto no_ptp;
7612
7613         req->port_id = cpu_to_le16(bp->pf.port_id);
7614         resp = hwrm_req_hold(bp, req);
7615         rc = hwrm_req_send(bp, req);
7616         if (rc)
7617                 goto exit;
7618
7619         flags = resp->flags;
7620         if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7621                 rc = -ENODEV;
7622                 goto exit;
7623         }
7624         if (!ptp) {
7625                 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7626                 if (!ptp) {
7627                         rc = -ENOMEM;
7628                         goto exit;
7629                 }
7630                 ptp->bp = bp;
7631                 bp->ptp_cfg = ptp;
7632         }
7633         if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7634                 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7635                 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7636         } else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7637                 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7638                 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7639         } else {
7640                 rc = -ENODEV;
7641                 goto exit;
7642         }
7643         phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
7644         rc = bnxt_ptp_init(bp, phc_cfg);
7645         if (rc)
7646                 netdev_warn(bp->dev, "PTP initialization failed.\n");
7647 exit:
7648         hwrm_req_drop(bp, req);
7649         if (!rc)
7650                 return 0;
7651
7652 no_ptp:
7653         bnxt_ptp_clear(bp);
7654         kfree(ptp);
7655         bp->ptp_cfg = NULL;
7656         return rc;
7657 }
7658
7659 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7660 {
7661         struct hwrm_func_qcaps_output *resp;
7662         struct hwrm_func_qcaps_input *req;
7663         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7664         u32 flags, flags_ext, flags_ext2;
7665         int rc;
7666
7667         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
7668         if (rc)
7669                 return rc;
7670
7671         req->fid = cpu_to_le16(0xffff);
7672         resp = hwrm_req_hold(bp, req);
7673         rc = hwrm_req_send(bp, req);
7674         if (rc)
7675                 goto hwrm_func_qcaps_exit;
7676
7677         flags = le32_to_cpu(resp->flags);
7678         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7679                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7680         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7681                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7682         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7683                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7684         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7685                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7686         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7687                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7688         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7689                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7690         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7691                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7692         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7693                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7694         if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
7695                 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
7696
7697         flags_ext = le32_to_cpu(resp->flags_ext);
7698         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7699                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7700         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
7701                 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
7702         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
7703                 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
7704         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
7705                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
7706         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
7707                 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
7708
7709         flags_ext2 = le32_to_cpu(resp->flags_ext2);
7710         if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
7711                 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
7712
7713         bp->tx_push_thresh = 0;
7714         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7715             BNXT_FW_MAJ(bp) > 217)
7716                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7717
7718         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7719         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7720         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7721         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7722         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7723         if (!hw_resc->max_hw_ring_grps)
7724                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7725         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7726         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7727         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7728
7729         if (BNXT_PF(bp)) {
7730                 struct bnxt_pf_info *pf = &bp->pf;
7731
7732                 pf->fw_fid = le16_to_cpu(resp->fid);
7733                 pf->port_id = le16_to_cpu(resp->port_id);
7734                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7735                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7736                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7737                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7738                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7739                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7740                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7741                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7742                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7743                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7744                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7745                         bp->flags |= BNXT_FLAG_WOL_CAP;
7746                 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
7747                         bp->fw_cap |= BNXT_FW_CAP_PTP;
7748                 } else {
7749                         bnxt_ptp_clear(bp);
7750                         kfree(bp->ptp_cfg);
7751                         bp->ptp_cfg = NULL;
7752                 }
7753         } else {
7754 #ifdef CONFIG_BNXT_SRIOV
7755                 struct bnxt_vf_info *vf = &bp->vf;
7756
7757                 vf->fw_fid = le16_to_cpu(resp->fid);
7758                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7759 #endif
7760         }
7761
7762 hwrm_func_qcaps_exit:
7763         hwrm_req_drop(bp, req);
7764         return rc;
7765 }
7766
7767 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
7768 {
7769         struct hwrm_dbg_qcaps_output *resp;
7770         struct hwrm_dbg_qcaps_input *req;
7771         int rc;
7772
7773         bp->fw_dbg_cap = 0;
7774         if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
7775                 return;
7776
7777         rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
7778         if (rc)
7779                 return;
7780
7781         req->fid = cpu_to_le16(0xffff);
7782         resp = hwrm_req_hold(bp, req);
7783         rc = hwrm_req_send(bp, req);
7784         if (rc)
7785                 goto hwrm_dbg_qcaps_exit;
7786
7787         bp->fw_dbg_cap = le32_to_cpu(resp->flags);
7788
7789 hwrm_dbg_qcaps_exit:
7790         hwrm_req_drop(bp, req);
7791 }
7792
7793 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7794
7795 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7796 {
7797         int rc;
7798
7799         rc = __bnxt_hwrm_func_qcaps(bp);
7800         if (rc)
7801                 return rc;
7802
7803         bnxt_hwrm_dbg_qcaps(bp);
7804
7805         rc = bnxt_hwrm_queue_qportcfg(bp);
7806         if (rc) {
7807                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7808                 return rc;
7809         }
7810         if (bp->hwrm_spec_code >= 0x10803) {
7811                 rc = bnxt_alloc_ctx_mem(bp);
7812                 if (rc)
7813                         return rc;
7814                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7815                 if (!rc)
7816                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7817         }
7818         return 0;
7819 }
7820
7821 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7822 {
7823         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7824         struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
7825         u32 flags;
7826         int rc;
7827
7828         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7829                 return 0;
7830
7831         rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
7832         if (rc)
7833                 return rc;
7834
7835         resp = hwrm_req_hold(bp, req);
7836         rc = hwrm_req_send(bp, req);
7837         if (rc)
7838                 goto hwrm_cfa_adv_qcaps_exit;
7839
7840         flags = le32_to_cpu(resp->flags);
7841         if (flags &
7842             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7843                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7844
7845 hwrm_cfa_adv_qcaps_exit:
7846         hwrm_req_drop(bp, req);
7847         return rc;
7848 }
7849
7850 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7851 {
7852         if (bp->fw_health)
7853                 return 0;
7854
7855         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7856         if (!bp->fw_health)
7857                 return -ENOMEM;
7858
7859         mutex_init(&bp->fw_health->lock);
7860         return 0;
7861 }
7862
7863 static int bnxt_alloc_fw_health(struct bnxt *bp)
7864 {
7865         int rc;
7866
7867         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7868             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7869                 return 0;
7870
7871         rc = __bnxt_alloc_fw_health(bp);
7872         if (rc) {
7873                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7874                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7875                 return rc;
7876         }
7877
7878         return 0;
7879 }
7880
7881 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7882 {
7883         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7884                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7885                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7886 }
7887
7888 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7889 {
7890         struct bnxt_fw_health *fw_health = bp->fw_health;
7891         u32 reg_type;
7892
7893         if (!fw_health)
7894                 return;
7895
7896         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7897         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7898                 fw_health->status_reliable = false;
7899
7900         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
7901         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7902                 fw_health->resets_reliable = false;
7903 }
7904
7905 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7906 {
7907         void __iomem *hs;
7908         u32 status_loc;
7909         u32 reg_type;
7910         u32 sig;
7911
7912         if (bp->fw_health)
7913                 bp->fw_health->status_reliable = false;
7914
7915         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7916         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7917
7918         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7919         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7920                 if (!bp->chip_num) {
7921                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7922                         bp->chip_num = readl(bp->bar0 +
7923                                              BNXT_FW_HEALTH_WIN_BASE +
7924                                              BNXT_GRC_REG_CHIP_NUM);
7925                 }
7926                 if (!BNXT_CHIP_P5(bp))
7927                         return;
7928
7929                 status_loc = BNXT_GRC_REG_STATUS_P5 |
7930                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
7931         } else {
7932                 status_loc = readl(hs + offsetof(struct hcomm_status,
7933                                                  fw_status_loc));
7934         }
7935
7936         if (__bnxt_alloc_fw_health(bp)) {
7937                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7938                 return;
7939         }
7940
7941         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7942         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7943         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7944                 __bnxt_map_fw_health_reg(bp, status_loc);
7945                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7946                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7947         }
7948
7949         bp->fw_health->status_reliable = true;
7950 }
7951
7952 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7953 {
7954         struct bnxt_fw_health *fw_health = bp->fw_health;
7955         u32 reg_base = 0xffffffff;
7956         int i;
7957
7958         bp->fw_health->status_reliable = false;
7959         bp->fw_health->resets_reliable = false;
7960         /* Only pre-map the monitoring GRC registers using window 3 */
7961         for (i = 0; i < 4; i++) {
7962                 u32 reg = fw_health->regs[i];
7963
7964                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7965                         continue;
7966                 if (reg_base == 0xffffffff)
7967                         reg_base = reg & BNXT_GRC_BASE_MASK;
7968                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7969                         return -ERANGE;
7970                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7971         }
7972         bp->fw_health->status_reliable = true;
7973         bp->fw_health->resets_reliable = true;
7974         if (reg_base == 0xffffffff)
7975                 return 0;
7976
7977         __bnxt_map_fw_health_reg(bp, reg_base);
7978         return 0;
7979 }
7980
7981 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
7982 {
7983         if (!bp->fw_health)
7984                 return;
7985
7986         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
7987                 bp->fw_health->status_reliable = true;
7988                 bp->fw_health->resets_reliable = true;
7989         } else {
7990                 bnxt_try_map_fw_health_reg(bp);
7991         }
7992 }
7993
7994 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7995 {
7996         struct bnxt_fw_health *fw_health = bp->fw_health;
7997         struct hwrm_error_recovery_qcfg_output *resp;
7998         struct hwrm_error_recovery_qcfg_input *req;
7999         int rc, i;
8000
8001         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8002                 return 0;
8003
8004         rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8005         if (rc)
8006                 return rc;
8007
8008         resp = hwrm_req_hold(bp, req);
8009         rc = hwrm_req_send(bp, req);
8010         if (rc)
8011                 goto err_recovery_out;
8012         fw_health->flags = le32_to_cpu(resp->flags);
8013         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8014             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8015                 rc = -EINVAL;
8016                 goto err_recovery_out;
8017         }
8018         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
8019         fw_health->master_func_wait_dsecs =
8020                 le32_to_cpu(resp->master_func_wait_period);
8021         fw_health->normal_func_wait_dsecs =
8022                 le32_to_cpu(resp->normal_func_wait_period);
8023         fw_health->post_reset_wait_dsecs =
8024                 le32_to_cpu(resp->master_func_wait_period_after_reset);
8025         fw_health->post_reset_max_wait_dsecs =
8026                 le32_to_cpu(resp->max_bailout_time_after_reset);
8027         fw_health->regs[BNXT_FW_HEALTH_REG] =
8028                 le32_to_cpu(resp->fw_health_status_reg);
8029         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
8030                 le32_to_cpu(resp->fw_heartbeat_reg);
8031         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
8032                 le32_to_cpu(resp->fw_reset_cnt_reg);
8033         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
8034                 le32_to_cpu(resp->reset_inprogress_reg);
8035         fw_health->fw_reset_inprog_reg_mask =
8036                 le32_to_cpu(resp->reset_inprogress_reg_mask);
8037         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
8038         if (fw_health->fw_reset_seq_cnt >= 16) {
8039                 rc = -EINVAL;
8040                 goto err_recovery_out;
8041         }
8042         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
8043                 fw_health->fw_reset_seq_regs[i] =
8044                         le32_to_cpu(resp->reset_reg[i]);
8045                 fw_health->fw_reset_seq_vals[i] =
8046                         le32_to_cpu(resp->reset_reg_val[i]);
8047                 fw_health->fw_reset_seq_delay_msec[i] =
8048                         resp->delay_after_reset[i];
8049         }
8050 err_recovery_out:
8051         hwrm_req_drop(bp, req);
8052         if (!rc)
8053                 rc = bnxt_map_fw_health_regs(bp);
8054         if (rc)
8055                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8056         return rc;
8057 }
8058
8059 static int bnxt_hwrm_func_reset(struct bnxt *bp)
8060 {
8061         struct hwrm_func_reset_input *req;
8062         int rc;
8063
8064         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
8065         if (rc)
8066                 return rc;
8067
8068         req->enables = 0;
8069         hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
8070         return hwrm_req_send(bp, req);
8071 }
8072
8073 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
8074 {
8075         struct hwrm_nvm_get_dev_info_output nvm_info;
8076
8077         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
8078                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
8079                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
8080                          nvm_info.nvm_cfg_ver_upd);
8081 }
8082
8083 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
8084 {
8085         struct hwrm_queue_qportcfg_output *resp;
8086         struct hwrm_queue_qportcfg_input *req;
8087         u8 i, j, *qptr;
8088         bool no_rdma;
8089         int rc = 0;
8090
8091         rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
8092         if (rc)
8093                 return rc;
8094
8095         resp = hwrm_req_hold(bp, req);
8096         rc = hwrm_req_send(bp, req);
8097         if (rc)
8098                 goto qportcfg_exit;
8099
8100         if (!resp->max_configurable_queues) {
8101                 rc = -EINVAL;
8102                 goto qportcfg_exit;
8103         }
8104         bp->max_tc = resp->max_configurable_queues;
8105         bp->max_lltc = resp->max_configurable_lossless_queues;
8106         if (bp->max_tc > BNXT_MAX_QUEUE)
8107                 bp->max_tc = BNXT_MAX_QUEUE;
8108
8109         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
8110         qptr = &resp->queue_id0;
8111         for (i = 0, j = 0; i < bp->max_tc; i++) {
8112                 bp->q_info[j].queue_id = *qptr;
8113                 bp->q_ids[i] = *qptr++;
8114                 bp->q_info[j].queue_profile = *qptr++;
8115                 bp->tc_to_qidx[j] = j;
8116                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
8117                     (no_rdma && BNXT_PF(bp)))
8118                         j++;
8119         }
8120         bp->max_q = bp->max_tc;
8121         bp->max_tc = max_t(u8, j, 1);
8122
8123         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
8124                 bp->max_tc = 1;
8125
8126         if (bp->max_lltc > bp->max_tc)
8127                 bp->max_lltc = bp->max_tc;
8128
8129 qportcfg_exit:
8130         hwrm_req_drop(bp, req);
8131         return rc;
8132 }
8133
8134 static int bnxt_hwrm_poll(struct bnxt *bp)
8135 {
8136         struct hwrm_ver_get_input *req;
8137         int rc;
8138
8139         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8140         if (rc)
8141                 return rc;
8142
8143         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8144         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8145         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8146
8147         hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
8148         rc = hwrm_req_send(bp, req);
8149         return rc;
8150 }
8151
8152 static int bnxt_hwrm_ver_get(struct bnxt *bp)
8153 {
8154         struct hwrm_ver_get_output *resp;
8155         struct hwrm_ver_get_input *req;
8156         u16 fw_maj, fw_min, fw_bld, fw_rsv;
8157         u32 dev_caps_cfg, hwrm_ver;
8158         int rc, len;
8159
8160         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8161         if (rc)
8162                 return rc;
8163
8164         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
8165         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
8166         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8167         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8168         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8169
8170         resp = hwrm_req_hold(bp, req);
8171         rc = hwrm_req_send(bp, req);
8172         if (rc)
8173                 goto hwrm_ver_get_exit;
8174
8175         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
8176
8177         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
8178                              resp->hwrm_intf_min_8b << 8 |
8179                              resp->hwrm_intf_upd_8b;
8180         if (resp->hwrm_intf_maj_8b < 1) {
8181                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
8182                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8183                             resp->hwrm_intf_upd_8b);
8184                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
8185         }
8186
8187         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
8188                         HWRM_VERSION_UPDATE;
8189
8190         if (bp->hwrm_spec_code > hwrm_ver)
8191                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8192                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
8193                          HWRM_VERSION_UPDATE);
8194         else
8195                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8196                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8197                          resp->hwrm_intf_upd_8b);
8198
8199         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
8200         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
8201                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
8202                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
8203                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
8204                 len = FW_VER_STR_LEN;
8205         } else {
8206                 fw_maj = resp->hwrm_fw_maj_8b;
8207                 fw_min = resp->hwrm_fw_min_8b;
8208                 fw_bld = resp->hwrm_fw_bld_8b;
8209                 fw_rsv = resp->hwrm_fw_rsvd_8b;
8210                 len = BC_HWRM_STR_LEN;
8211         }
8212         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
8213         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
8214                  fw_rsv);
8215
8216         if (strlen(resp->active_pkg_name)) {
8217                 int fw_ver_len = strlen(bp->fw_ver_str);
8218
8219                 snprintf(bp->fw_ver_str + fw_ver_len,
8220                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
8221                          resp->active_pkg_name);
8222                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
8223         }
8224
8225         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
8226         if (!bp->hwrm_cmd_timeout)
8227                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
8228         bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
8229         if (!bp->hwrm_cmd_max_timeout)
8230                 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
8231         else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
8232                 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
8233                             bp->hwrm_cmd_max_timeout / 1000);
8234
8235         if (resp->hwrm_intf_maj_8b >= 1) {
8236                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
8237                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
8238         }
8239         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
8240                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
8241
8242         bp->chip_num = le16_to_cpu(resp->chip_num);
8243         bp->chip_rev = resp->chip_rev;
8244         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
8245             !resp->chip_metal)
8246                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
8247
8248         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
8249         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
8250             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
8251                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
8252
8253         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
8254                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
8255
8256         if (dev_caps_cfg &
8257             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
8258                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
8259
8260         if (dev_caps_cfg &
8261             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
8262                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
8263
8264         if (dev_caps_cfg &
8265             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
8266                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
8267
8268 hwrm_ver_get_exit:
8269         hwrm_req_drop(bp, req);
8270         return rc;
8271 }
8272
8273 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
8274 {
8275         struct hwrm_fw_set_time_input *req;
8276         struct tm tm;
8277         time64_t now = ktime_get_real_seconds();
8278         int rc;
8279
8280         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
8281             bp->hwrm_spec_code < 0x10400)
8282                 return -EOPNOTSUPP;
8283
8284         time64_to_tm(now, 0, &tm);
8285         rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
8286         if (rc)
8287                 return rc;
8288
8289         req->year = cpu_to_le16(1900 + tm.tm_year);
8290         req->month = 1 + tm.tm_mon;
8291         req->day = tm.tm_mday;
8292         req->hour = tm.tm_hour;
8293         req->minute = tm.tm_min;
8294         req->second = tm.tm_sec;
8295         return hwrm_req_send(bp, req);
8296 }
8297
8298 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
8299 {
8300         u64 sw_tmp;
8301
8302         hw &= mask;
8303         sw_tmp = (*sw & ~mask) | hw;
8304         if (hw < (*sw & mask))
8305                 sw_tmp += mask + 1;
8306         WRITE_ONCE(*sw, sw_tmp);
8307 }
8308
8309 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8310                                     int count, bool ignore_zero)
8311 {
8312         int i;
8313
8314         for (i = 0; i < count; i++) {
8315                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8316
8317                 if (ignore_zero && !hw)
8318                         continue;
8319
8320                 if (masks[i] == -1ULL)
8321                         sw_stats[i] = hw;
8322                 else
8323                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8324         }
8325 }
8326
8327 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8328 {
8329         if (!stats->hw_stats)
8330                 return;
8331
8332         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8333                                 stats->hw_masks, stats->len / 8, false);
8334 }
8335
8336 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8337 {
8338         struct bnxt_stats_mem *ring0_stats;
8339         bool ignore_zero = false;
8340         int i;
8341
8342         /* Chip bug.  Counter intermittently becomes 0. */
8343         if (bp->flags & BNXT_FLAG_CHIP_P5)
8344                 ignore_zero = true;
8345
8346         for (i = 0; i < bp->cp_nr_rings; i++) {
8347                 struct bnxt_napi *bnapi = bp->bnapi[i];
8348                 struct bnxt_cp_ring_info *cpr;
8349                 struct bnxt_stats_mem *stats;
8350
8351                 cpr = &bnapi->cp_ring;
8352                 stats = &cpr->stats;
8353                 if (!i)
8354                         ring0_stats = stats;
8355                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8356                                         ring0_stats->hw_masks,
8357                                         ring0_stats->len / 8, ignore_zero);
8358         }
8359         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8360                 struct bnxt_stats_mem *stats = &bp->port_stats;
8361                 __le64 *hw_stats = stats->hw_stats;
8362                 u64 *sw_stats = stats->sw_stats;
8363                 u64 *masks = stats->hw_masks;
8364                 int cnt;
8365
8366                 cnt = sizeof(struct rx_port_stats) / 8;
8367                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8368
8369                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8370                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8371                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8372                 cnt = sizeof(struct tx_port_stats) / 8;
8373                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8374         }
8375         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8376                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8377                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8378         }
8379 }
8380
8381 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8382 {
8383         struct hwrm_port_qstats_input *req;
8384         struct bnxt_pf_info *pf = &bp->pf;
8385         int rc;
8386
8387         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8388                 return 0;
8389
8390         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8391                 return -EOPNOTSUPP;
8392
8393         rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
8394         if (rc)
8395                 return rc;
8396
8397         req->flags = flags;
8398         req->port_id = cpu_to_le16(pf->port_id);
8399         req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8400                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
8401         req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8402         return hwrm_req_send(bp, req);
8403 }
8404
8405 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8406 {
8407         struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
8408         struct hwrm_queue_pri2cos_qcfg_input *req_qc;
8409         struct hwrm_port_qstats_ext_output *resp_qs;
8410         struct hwrm_port_qstats_ext_input *req_qs;
8411         struct bnxt_pf_info *pf = &bp->pf;
8412         u32 tx_stat_size;
8413         int rc;
8414
8415         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8416                 return 0;
8417
8418         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8419                 return -EOPNOTSUPP;
8420
8421         rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
8422         if (rc)
8423                 return rc;
8424
8425         req_qs->flags = flags;
8426         req_qs->port_id = cpu_to_le16(pf->port_id);
8427         req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8428         req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8429         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8430                        sizeof(struct tx_port_stats_ext) : 0;
8431         req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
8432         req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8433         resp_qs = hwrm_req_hold(bp, req_qs);
8434         rc = hwrm_req_send(bp, req_qs);
8435         if (!rc) {
8436                 bp->fw_rx_stats_ext_size =
8437                         le16_to_cpu(resp_qs->rx_stat_size) / 8;
8438                 if (BNXT_FW_MAJ(bp) < 220 &&
8439                     bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
8440                         bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
8441
8442                 bp->fw_tx_stats_ext_size = tx_stat_size ?
8443                         le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
8444         } else {
8445                 bp->fw_rx_stats_ext_size = 0;
8446                 bp->fw_tx_stats_ext_size = 0;
8447         }
8448         hwrm_req_drop(bp, req_qs);
8449
8450         if (flags)
8451                 return rc;
8452
8453         if (bp->fw_tx_stats_ext_size <=
8454             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8455                 bp->pri2cos_valid = 0;
8456                 return rc;
8457         }
8458
8459         rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
8460         if (rc)
8461                 return rc;
8462
8463         req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8464
8465         resp_qc = hwrm_req_hold(bp, req_qc);
8466         rc = hwrm_req_send(bp, req_qc);
8467         if (!rc) {
8468                 u8 *pri2cos;
8469                 int i, j;
8470
8471                 pri2cos = &resp_qc->pri0_cos_queue_id;
8472                 for (i = 0; i < 8; i++) {
8473                         u8 queue_id = pri2cos[i];
8474                         u8 queue_idx;
8475
8476                         /* Per port queue IDs start from 0, 10, 20, etc */
8477                         queue_idx = queue_id % 10;
8478                         if (queue_idx > BNXT_MAX_QUEUE) {
8479                                 bp->pri2cos_valid = false;
8480                                 hwrm_req_drop(bp, req_qc);
8481                                 return rc;
8482                         }
8483                         for (j = 0; j < bp->max_q; j++) {
8484                                 if (bp->q_ids[j] == queue_id)
8485                                         bp->pri2cos_idx[i] = queue_idx;
8486                         }
8487                 }
8488                 bp->pri2cos_valid = true;
8489         }
8490         hwrm_req_drop(bp, req_qc);
8491
8492         return rc;
8493 }
8494
8495 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8496 {
8497         bnxt_hwrm_tunnel_dst_port_free(bp,
8498                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8499         bnxt_hwrm_tunnel_dst_port_free(bp,
8500                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8501 }
8502
8503 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8504 {
8505         int rc, i;
8506         u32 tpa_flags = 0;
8507
8508         if (set_tpa)
8509                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
8510         else if (BNXT_NO_FW_ACCESS(bp))
8511                 return 0;
8512         for (i = 0; i < bp->nr_vnics; i++) {
8513                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8514                 if (rc) {
8515                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8516                                    i, rc);
8517                         return rc;
8518                 }
8519         }
8520         return 0;
8521 }
8522
8523 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8524 {
8525         int i;
8526
8527         for (i = 0; i < bp->nr_vnics; i++)
8528                 bnxt_hwrm_vnic_set_rss(bp, i, false);
8529 }
8530
8531 static void bnxt_clear_vnic(struct bnxt *bp)
8532 {
8533         if (!bp->vnic_info)
8534                 return;
8535
8536         bnxt_hwrm_clear_vnic_filter(bp);
8537         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8538                 /* clear all RSS setting before free vnic ctx */
8539                 bnxt_hwrm_clear_vnic_rss(bp);
8540                 bnxt_hwrm_vnic_ctx_free(bp);
8541         }
8542         /* before free the vnic, undo the vnic tpa settings */
8543         if (bp->flags & BNXT_FLAG_TPA)
8544                 bnxt_set_tpa(bp, false);
8545         bnxt_hwrm_vnic_free(bp);
8546         if (bp->flags & BNXT_FLAG_CHIP_P5)
8547                 bnxt_hwrm_vnic_ctx_free(bp);
8548 }
8549
8550 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8551                                     bool irq_re_init)
8552 {
8553         bnxt_clear_vnic(bp);
8554         bnxt_hwrm_ring_free(bp, close_path);
8555         bnxt_hwrm_ring_grp_free(bp);
8556         if (irq_re_init) {
8557                 bnxt_hwrm_stat_ctx_free(bp);
8558                 bnxt_hwrm_free_tunnel_ports(bp);
8559         }
8560 }
8561
8562 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8563 {
8564         struct hwrm_func_cfg_input *req;
8565         u8 evb_mode;
8566         int rc;
8567
8568         if (br_mode == BRIDGE_MODE_VEB)
8569                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8570         else if (br_mode == BRIDGE_MODE_VEPA)
8571                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8572         else
8573                 return -EINVAL;
8574
8575         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8576         if (rc)
8577                 return rc;
8578
8579         req->fid = cpu_to_le16(0xffff);
8580         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8581         req->evb_mode = evb_mode;
8582         return hwrm_req_send(bp, req);
8583 }
8584
8585 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8586 {
8587         struct hwrm_func_cfg_input *req;
8588         int rc;
8589
8590         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8591                 return 0;
8592
8593         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8594         if (rc)
8595                 return rc;
8596
8597         req->fid = cpu_to_le16(0xffff);
8598         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8599         req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8600         if (size == 128)
8601                 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8602
8603         return hwrm_req_send(bp, req);
8604 }
8605
8606 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8607 {
8608         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8609         int rc;
8610
8611         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8612                 goto skip_rss_ctx;
8613
8614         /* allocate context for vnic */
8615         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8616         if (rc) {
8617                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8618                            vnic_id, rc);
8619                 goto vnic_setup_err;
8620         }
8621         bp->rsscos_nr_ctxs++;
8622
8623         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8624                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8625                 if (rc) {
8626                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8627                                    vnic_id, rc);
8628                         goto vnic_setup_err;
8629                 }
8630                 bp->rsscos_nr_ctxs++;
8631         }
8632
8633 skip_rss_ctx:
8634         /* configure default vnic, ring grp */
8635         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8636         if (rc) {
8637                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8638                            vnic_id, rc);
8639                 goto vnic_setup_err;
8640         }
8641
8642         /* Enable RSS hashing on vnic */
8643         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8644         if (rc) {
8645                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8646                            vnic_id, rc);
8647                 goto vnic_setup_err;
8648         }
8649
8650         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8651                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8652                 if (rc) {
8653                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8654                                    vnic_id, rc);
8655                 }
8656         }
8657
8658 vnic_setup_err:
8659         return rc;
8660 }
8661
8662 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8663 {
8664         int rc, i, nr_ctxs;
8665
8666         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8667         for (i = 0; i < nr_ctxs; i++) {
8668                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8669                 if (rc) {
8670                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8671                                    vnic_id, i, rc);
8672                         break;
8673                 }
8674                 bp->rsscos_nr_ctxs++;
8675         }
8676         if (i < nr_ctxs)
8677                 return -ENOMEM;
8678
8679         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8680         if (rc) {
8681                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8682                            vnic_id, rc);
8683                 return rc;
8684         }
8685         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8686         if (rc) {
8687                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8688                            vnic_id, rc);
8689                 return rc;
8690         }
8691         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8692                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8693                 if (rc) {
8694                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8695                                    vnic_id, rc);
8696                 }
8697         }
8698         return rc;
8699 }
8700
8701 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8702 {
8703         if (bp->flags & BNXT_FLAG_CHIP_P5)
8704                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8705         else
8706                 return __bnxt_setup_vnic(bp, vnic_id);
8707 }
8708
8709 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8710 {
8711 #ifdef CONFIG_RFS_ACCEL
8712         int i, rc = 0;
8713
8714         if (bp->flags & BNXT_FLAG_CHIP_P5)
8715                 return 0;
8716
8717         for (i = 0; i < bp->rx_nr_rings; i++) {
8718                 struct bnxt_vnic_info *vnic;
8719                 u16 vnic_id = i + 1;
8720                 u16 ring_id = i;
8721
8722                 if (vnic_id >= bp->nr_vnics)
8723                         break;
8724
8725                 vnic = &bp->vnic_info[vnic_id];
8726                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8727                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8728                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8729                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8730                 if (rc) {
8731                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8732                                    vnic_id, rc);
8733                         break;
8734                 }
8735                 rc = bnxt_setup_vnic(bp, vnic_id);
8736                 if (rc)
8737                         break;
8738         }
8739         return rc;
8740 #else
8741         return 0;
8742 #endif
8743 }
8744
8745 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8746 static bool bnxt_promisc_ok(struct bnxt *bp)
8747 {
8748 #ifdef CONFIG_BNXT_SRIOV
8749         if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8750                 return false;
8751 #endif
8752         return true;
8753 }
8754
8755 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8756 {
8757         unsigned int rc = 0;
8758
8759         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8760         if (rc) {
8761                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8762                            rc);
8763                 return rc;
8764         }
8765
8766         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8767         if (rc) {
8768                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8769                            rc);
8770                 return rc;
8771         }
8772         return rc;
8773 }
8774
8775 static int bnxt_cfg_rx_mode(struct bnxt *);
8776 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8777
8778 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8779 {
8780         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8781         int rc = 0;
8782         unsigned int rx_nr_rings = bp->rx_nr_rings;
8783
8784         if (irq_re_init) {
8785                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8786                 if (rc) {
8787                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8788                                    rc);
8789                         goto err_out;
8790                 }
8791         }
8792
8793         rc = bnxt_hwrm_ring_alloc(bp);
8794         if (rc) {
8795                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8796                 goto err_out;
8797         }
8798
8799         rc = bnxt_hwrm_ring_grp_alloc(bp);
8800         if (rc) {
8801                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8802                 goto err_out;
8803         }
8804
8805         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8806                 rx_nr_rings--;
8807
8808         /* default vnic 0 */
8809         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8810         if (rc) {
8811                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8812                 goto err_out;
8813         }
8814
8815         rc = bnxt_setup_vnic(bp, 0);
8816         if (rc)
8817                 goto err_out;
8818         if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
8819                 bnxt_hwrm_update_rss_hash_cfg(bp);
8820
8821         if (bp->flags & BNXT_FLAG_RFS) {
8822                 rc = bnxt_alloc_rfs_vnics(bp);
8823                 if (rc)
8824                         goto err_out;
8825         }
8826
8827         if (bp->flags & BNXT_FLAG_TPA) {
8828                 rc = bnxt_set_tpa(bp, true);
8829                 if (rc)
8830                         goto err_out;
8831         }
8832
8833         if (BNXT_VF(bp))
8834                 bnxt_update_vf_mac(bp);
8835
8836         /* Filter for default vnic 0 */
8837         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8838         if (rc) {
8839                 if (BNXT_VF(bp) && rc == -ENODEV)
8840                         netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
8841                 else
8842                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8843                 goto err_out;
8844         }
8845         vnic->uc_filter_count = 1;
8846
8847         vnic->rx_mask = 0;
8848         if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
8849                 goto skip_rx_mask;
8850
8851         if (bp->dev->flags & IFF_BROADCAST)
8852                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8853
8854         if (bp->dev->flags & IFF_PROMISC)
8855                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8856
8857         if (bp->dev->flags & IFF_ALLMULTI) {
8858                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8859                 vnic->mc_list_count = 0;
8860         } else if (bp->dev->flags & IFF_MULTICAST) {
8861                 u32 mask = 0;
8862
8863                 bnxt_mc_list_updated(bp, &mask);
8864                 vnic->rx_mask |= mask;
8865         }
8866
8867         rc = bnxt_cfg_rx_mode(bp);
8868         if (rc)
8869                 goto err_out;
8870
8871 skip_rx_mask:
8872         rc = bnxt_hwrm_set_coal(bp);
8873         if (rc)
8874                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8875                                 rc);
8876
8877         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8878                 rc = bnxt_setup_nitroa0_vnic(bp);
8879                 if (rc)
8880                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8881                                    rc);
8882         }
8883
8884         if (BNXT_VF(bp)) {
8885                 bnxt_hwrm_func_qcfg(bp);
8886                 netdev_update_features(bp->dev);
8887         }
8888
8889         return 0;
8890
8891 err_out:
8892         bnxt_hwrm_resource_free(bp, 0, true);
8893
8894         return rc;
8895 }
8896
8897 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8898 {
8899         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8900         return 0;
8901 }
8902
8903 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8904 {
8905         bnxt_init_cp_rings(bp);
8906         bnxt_init_rx_rings(bp);
8907         bnxt_init_tx_rings(bp);
8908         bnxt_init_ring_grps(bp, irq_re_init);
8909         bnxt_init_vnics(bp);
8910
8911         return bnxt_init_chip(bp, irq_re_init);
8912 }
8913
8914 static int bnxt_set_real_num_queues(struct bnxt *bp)
8915 {
8916         int rc;
8917         struct net_device *dev = bp->dev;
8918
8919         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8920                                           bp->tx_nr_rings_xdp);
8921         if (rc)
8922                 return rc;
8923
8924         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8925         if (rc)
8926                 return rc;
8927
8928 #ifdef CONFIG_RFS_ACCEL
8929         if (bp->flags & BNXT_FLAG_RFS)
8930                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8931 #endif
8932
8933         return rc;
8934 }
8935
8936 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8937                            bool shared)
8938 {
8939         int _rx = *rx, _tx = *tx;
8940
8941         if (shared) {
8942                 *rx = min_t(int, _rx, max);
8943                 *tx = min_t(int, _tx, max);
8944         } else {
8945                 if (max < 2)
8946                         return -ENOMEM;
8947
8948                 while (_rx + _tx > max) {
8949                         if (_rx > _tx && _rx > 1)
8950                                 _rx--;
8951                         else if (_tx > 1)
8952                                 _tx--;
8953                 }
8954                 *rx = _rx;
8955                 *tx = _tx;
8956         }
8957         return 0;
8958 }
8959
8960 static void bnxt_setup_msix(struct bnxt *bp)
8961 {
8962         const int len = sizeof(bp->irq_tbl[0].name);
8963         struct net_device *dev = bp->dev;
8964         int tcs, i;
8965
8966         tcs = netdev_get_num_tc(dev);
8967         if (tcs) {
8968                 int i, off, count;
8969
8970                 for (i = 0; i < tcs; i++) {
8971                         count = bp->tx_nr_rings_per_tc;
8972                         off = i * count;
8973                         netdev_set_tc_queue(dev, i, count, off);
8974                 }
8975         }
8976
8977         for (i = 0; i < bp->cp_nr_rings; i++) {
8978                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8979                 char *attr;
8980
8981                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8982                         attr = "TxRx";
8983                 else if (i < bp->rx_nr_rings)
8984                         attr = "rx";
8985                 else
8986                         attr = "tx";
8987
8988                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8989                          attr, i);
8990                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8991         }
8992 }
8993
8994 static void bnxt_setup_inta(struct bnxt *bp)
8995 {
8996         const int len = sizeof(bp->irq_tbl[0].name);
8997
8998         if (netdev_get_num_tc(bp->dev))
8999                 netdev_reset_tc(bp->dev);
9000
9001         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
9002                  0);
9003         bp->irq_tbl[0].handler = bnxt_inta;
9004 }
9005
9006 static int bnxt_init_int_mode(struct bnxt *bp);
9007
9008 static int bnxt_setup_int_mode(struct bnxt *bp)
9009 {
9010         int rc;
9011
9012         if (!bp->irq_tbl) {
9013                 rc = bnxt_init_int_mode(bp);
9014                 if (rc || !bp->irq_tbl)
9015                         return rc ?: -ENODEV;
9016         }
9017
9018         if (bp->flags & BNXT_FLAG_USING_MSIX)
9019                 bnxt_setup_msix(bp);
9020         else
9021                 bnxt_setup_inta(bp);
9022
9023         rc = bnxt_set_real_num_queues(bp);
9024         return rc;
9025 }
9026
9027 #ifdef CONFIG_RFS_ACCEL
9028 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
9029 {
9030         return bp->hw_resc.max_rsscos_ctxs;
9031 }
9032
9033 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
9034 {
9035         return bp->hw_resc.max_vnics;
9036 }
9037 #endif
9038
9039 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
9040 {
9041         return bp->hw_resc.max_stat_ctxs;
9042 }
9043
9044 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
9045 {
9046         return bp->hw_resc.max_cp_rings;
9047 }
9048
9049 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
9050 {
9051         unsigned int cp = bp->hw_resc.max_cp_rings;
9052
9053         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9054                 cp -= bnxt_get_ulp_msix_num(bp);
9055
9056         return cp;
9057 }
9058
9059 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
9060 {
9061         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9062
9063         if (bp->flags & BNXT_FLAG_CHIP_P5)
9064                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
9065
9066         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
9067 }
9068
9069 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
9070 {
9071         bp->hw_resc.max_irqs = max_irqs;
9072 }
9073
9074 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
9075 {
9076         unsigned int cp;
9077
9078         cp = bnxt_get_max_func_cp_rings_for_en(bp);
9079         if (bp->flags & BNXT_FLAG_CHIP_P5)
9080                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
9081         else
9082                 return cp - bp->cp_nr_rings;
9083 }
9084
9085 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
9086 {
9087         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
9088 }
9089
9090 int bnxt_get_avail_msix(struct bnxt *bp, int num)
9091 {
9092         int max_cp = bnxt_get_max_func_cp_rings(bp);
9093         int max_irq = bnxt_get_max_func_irqs(bp);
9094         int total_req = bp->cp_nr_rings + num;
9095         int max_idx, avail_msix;
9096
9097         max_idx = bp->total_irqs;
9098         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9099                 max_idx = min_t(int, bp->total_irqs, max_cp);
9100         avail_msix = max_idx - bp->cp_nr_rings;
9101         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
9102                 return avail_msix;
9103
9104         if (max_irq < total_req) {
9105                 num = max_irq - bp->cp_nr_rings;
9106                 if (num <= 0)
9107                         return 0;
9108         }
9109         return num;
9110 }
9111
9112 static int bnxt_get_num_msix(struct bnxt *bp)
9113 {
9114         if (!BNXT_NEW_RM(bp))
9115                 return bnxt_get_max_func_irqs(bp);
9116
9117         return bnxt_nq_rings_in_use(bp);
9118 }
9119
9120 static int bnxt_init_msix(struct bnxt *bp)
9121 {
9122         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
9123         struct msix_entry *msix_ent;
9124
9125         total_vecs = bnxt_get_num_msix(bp);
9126         max = bnxt_get_max_func_irqs(bp);
9127         if (total_vecs > max)
9128                 total_vecs = max;
9129
9130         if (!total_vecs)
9131                 return 0;
9132
9133         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
9134         if (!msix_ent)
9135                 return -ENOMEM;
9136
9137         for (i = 0; i < total_vecs; i++) {
9138                 msix_ent[i].entry = i;
9139                 msix_ent[i].vector = 0;
9140         }
9141
9142         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
9143                 min = 2;
9144
9145         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
9146         ulp_msix = bnxt_get_ulp_msix_num(bp);
9147         if (total_vecs < 0 || total_vecs < ulp_msix) {
9148                 rc = -ENODEV;
9149                 goto msix_setup_exit;
9150         }
9151
9152         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
9153         if (bp->irq_tbl) {
9154                 for (i = 0; i < total_vecs; i++)
9155                         bp->irq_tbl[i].vector = msix_ent[i].vector;
9156
9157                 bp->total_irqs = total_vecs;
9158                 /* Trim rings based upon num of vectors allocated */
9159                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
9160                                      total_vecs - ulp_msix, min == 1);
9161                 if (rc)
9162                         goto msix_setup_exit;
9163
9164                 bp->cp_nr_rings = (min == 1) ?
9165                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9166                                   bp->tx_nr_rings + bp->rx_nr_rings;
9167
9168         } else {
9169                 rc = -ENOMEM;
9170                 goto msix_setup_exit;
9171         }
9172         bp->flags |= BNXT_FLAG_USING_MSIX;
9173         kfree(msix_ent);
9174         return 0;
9175
9176 msix_setup_exit:
9177         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
9178         kfree(bp->irq_tbl);
9179         bp->irq_tbl = NULL;
9180         pci_disable_msix(bp->pdev);
9181         kfree(msix_ent);
9182         return rc;
9183 }
9184
9185 static int bnxt_init_inta(struct bnxt *bp)
9186 {
9187         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
9188         if (!bp->irq_tbl)
9189                 return -ENOMEM;
9190
9191         bp->total_irqs = 1;
9192         bp->rx_nr_rings = 1;
9193         bp->tx_nr_rings = 1;
9194         bp->cp_nr_rings = 1;
9195         bp->flags |= BNXT_FLAG_SHARED_RINGS;
9196         bp->irq_tbl[0].vector = bp->pdev->irq;
9197         return 0;
9198 }
9199
9200 static int bnxt_init_int_mode(struct bnxt *bp)
9201 {
9202         int rc = -ENODEV;
9203
9204         if (bp->flags & BNXT_FLAG_MSIX_CAP)
9205                 rc = bnxt_init_msix(bp);
9206
9207         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
9208                 /* fallback to INTA */
9209                 rc = bnxt_init_inta(bp);
9210         }
9211         return rc;
9212 }
9213
9214 static void bnxt_clear_int_mode(struct bnxt *bp)
9215 {
9216         if (bp->flags & BNXT_FLAG_USING_MSIX)
9217                 pci_disable_msix(bp->pdev);
9218
9219         kfree(bp->irq_tbl);
9220         bp->irq_tbl = NULL;
9221         bp->flags &= ~BNXT_FLAG_USING_MSIX;
9222 }
9223
9224 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
9225 {
9226         int tcs = netdev_get_num_tc(bp->dev);
9227         bool irq_cleared = false;
9228         int rc;
9229
9230         if (!bnxt_need_reserve_rings(bp))
9231                 return 0;
9232
9233         if (irq_re_init && BNXT_NEW_RM(bp) &&
9234             bnxt_get_num_msix(bp) != bp->total_irqs) {
9235                 bnxt_ulp_irq_stop(bp);
9236                 bnxt_clear_int_mode(bp);
9237                 irq_cleared = true;
9238         }
9239         rc = __bnxt_reserve_rings(bp);
9240         if (irq_cleared) {
9241                 if (!rc)
9242                         rc = bnxt_init_int_mode(bp);
9243                 bnxt_ulp_irq_restart(bp, rc);
9244         }
9245         if (rc) {
9246                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
9247                 return rc;
9248         }
9249         if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
9250                     bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
9251                 netdev_err(bp->dev, "tx ring reservation failure\n");
9252                 netdev_reset_tc(bp->dev);
9253                 if (bp->tx_nr_rings_xdp)
9254                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
9255                 else
9256                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9257                 return -ENOMEM;
9258         }
9259         return 0;
9260 }
9261
9262 static void bnxt_free_irq(struct bnxt *bp)
9263 {
9264         struct bnxt_irq *irq;
9265         int i;
9266
9267 #ifdef CONFIG_RFS_ACCEL
9268         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
9269         bp->dev->rx_cpu_rmap = NULL;
9270 #endif
9271         if (!bp->irq_tbl || !bp->bnapi)
9272                 return;
9273
9274         for (i = 0; i < bp->cp_nr_rings; i++) {
9275                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9276
9277                 irq = &bp->irq_tbl[map_idx];
9278                 if (irq->requested) {
9279                         if (irq->have_cpumask) {
9280                                 irq_set_affinity_hint(irq->vector, NULL);
9281                                 free_cpumask_var(irq->cpu_mask);
9282                                 irq->have_cpumask = 0;
9283                         }
9284                         free_irq(irq->vector, bp->bnapi[i]);
9285                 }
9286
9287                 irq->requested = 0;
9288         }
9289 }
9290
9291 static int bnxt_request_irq(struct bnxt *bp)
9292 {
9293         int i, j, rc = 0;
9294         unsigned long flags = 0;
9295 #ifdef CONFIG_RFS_ACCEL
9296         struct cpu_rmap *rmap;
9297 #endif
9298
9299         rc = bnxt_setup_int_mode(bp);
9300         if (rc) {
9301                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
9302                            rc);
9303                 return rc;
9304         }
9305 #ifdef CONFIG_RFS_ACCEL
9306         rmap = bp->dev->rx_cpu_rmap;
9307 #endif
9308         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
9309                 flags = IRQF_SHARED;
9310
9311         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
9312                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9313                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
9314
9315 #ifdef CONFIG_RFS_ACCEL
9316                 if (rmap && bp->bnapi[i]->rx_ring) {
9317                         rc = irq_cpu_rmap_add(rmap, irq->vector);
9318                         if (rc)
9319                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
9320                                             j);
9321                         j++;
9322                 }
9323 #endif
9324                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
9325                                  bp->bnapi[i]);
9326                 if (rc)
9327                         break;
9328
9329                 irq->requested = 1;
9330
9331                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
9332                         int numa_node = dev_to_node(&bp->pdev->dev);
9333
9334                         irq->have_cpumask = 1;
9335                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
9336                                         irq->cpu_mask);
9337                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
9338                         if (rc) {
9339                                 netdev_warn(bp->dev,
9340                                             "Set affinity failed, IRQ = %d\n",
9341                                             irq->vector);
9342                                 break;
9343                         }
9344                 }
9345         }
9346         return rc;
9347 }
9348
9349 static void bnxt_del_napi(struct bnxt *bp)
9350 {
9351         int i;
9352
9353         if (!bp->bnapi)
9354                 return;
9355
9356         for (i = 0; i < bp->cp_nr_rings; i++) {
9357                 struct bnxt_napi *bnapi = bp->bnapi[i];
9358
9359                 __netif_napi_del(&bnapi->napi);
9360         }
9361         /* We called __netif_napi_del(), we need
9362          * to respect an RCU grace period before freeing napi structures.
9363          */
9364         synchronize_net();
9365 }
9366
9367 static void bnxt_init_napi(struct bnxt *bp)
9368 {
9369         int i;
9370         unsigned int cp_nr_rings = bp->cp_nr_rings;
9371         struct bnxt_napi *bnapi;
9372
9373         if (bp->flags & BNXT_FLAG_USING_MSIX) {
9374                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9375
9376                 if (bp->flags & BNXT_FLAG_CHIP_P5)
9377                         poll_fn = bnxt_poll_p5;
9378                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9379                         cp_nr_rings--;
9380                 for (i = 0; i < cp_nr_rings; i++) {
9381                         bnapi = bp->bnapi[i];
9382                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
9383                 }
9384                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9385                         bnapi = bp->bnapi[cp_nr_rings];
9386                         netif_napi_add(bp->dev, &bnapi->napi,
9387                                        bnxt_poll_nitroa0);
9388                 }
9389         } else {
9390                 bnapi = bp->bnapi[0];
9391                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
9392         }
9393 }
9394
9395 static void bnxt_disable_napi(struct bnxt *bp)
9396 {
9397         int i;
9398
9399         if (!bp->bnapi ||
9400             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9401                 return;
9402
9403         for (i = 0; i < bp->cp_nr_rings; i++) {
9404                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
9405
9406                 napi_disable(&bp->bnapi[i]->napi);
9407                 if (bp->bnapi[i]->rx_ring)
9408                         cancel_work_sync(&cpr->dim.work);
9409         }
9410 }
9411
9412 static void bnxt_enable_napi(struct bnxt *bp)
9413 {
9414         int i;
9415
9416         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9417         for (i = 0; i < bp->cp_nr_rings; i++) {
9418                 struct bnxt_napi *bnapi = bp->bnapi[i];
9419                 struct bnxt_cp_ring_info *cpr;
9420
9421                 cpr = &bnapi->cp_ring;
9422                 if (bnapi->in_reset)
9423                         cpr->sw_stats.rx.rx_resets++;
9424                 bnapi->in_reset = false;
9425
9426                 if (bnapi->rx_ring) {
9427                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9428                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9429                 }
9430                 napi_enable(&bnapi->napi);
9431         }
9432 }
9433
9434 void bnxt_tx_disable(struct bnxt *bp)
9435 {
9436         int i;
9437         struct bnxt_tx_ring_info *txr;
9438
9439         if (bp->tx_ring) {
9440                 for (i = 0; i < bp->tx_nr_rings; i++) {
9441                         txr = &bp->tx_ring[i];
9442                         WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
9443                 }
9444         }
9445         /* Make sure napi polls see @dev_state change */
9446         synchronize_net();
9447         /* Drop carrier first to prevent TX timeout */
9448         netif_carrier_off(bp->dev);
9449         /* Stop all TX queues */
9450         netif_tx_disable(bp->dev);
9451 }
9452
9453 void bnxt_tx_enable(struct bnxt *bp)
9454 {
9455         int i;
9456         struct bnxt_tx_ring_info *txr;
9457
9458         for (i = 0; i < bp->tx_nr_rings; i++) {
9459                 txr = &bp->tx_ring[i];
9460                 WRITE_ONCE(txr->dev_state, 0);
9461         }
9462         /* Make sure napi polls see @dev_state change */
9463         synchronize_net();
9464         netif_tx_wake_all_queues(bp->dev);
9465         if (BNXT_LINK_IS_UP(bp))
9466                 netif_carrier_on(bp->dev);
9467 }
9468
9469 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9470 {
9471         u8 active_fec = link_info->active_fec_sig_mode &
9472                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9473
9474         switch (active_fec) {
9475         default:
9476         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9477                 return "None";
9478         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9479                 return "Clause 74 BaseR";
9480         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9481                 return "Clause 91 RS(528,514)";
9482         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9483                 return "Clause 91 RS544_1XN";
9484         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9485                 return "Clause 91 RS(544,514)";
9486         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9487                 return "Clause 91 RS272_1XN";
9488         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9489                 return "Clause 91 RS(272,257)";
9490         }
9491 }
9492
9493 void bnxt_report_link(struct bnxt *bp)
9494 {
9495         if (BNXT_LINK_IS_UP(bp)) {
9496                 const char *signal = "";
9497                 const char *flow_ctrl;
9498                 const char *duplex;
9499                 u32 speed;
9500                 u16 fec;
9501
9502                 netif_carrier_on(bp->dev);
9503                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9504                 if (speed == SPEED_UNKNOWN) {
9505                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9506                         return;
9507                 }
9508                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9509                         duplex = "full";
9510                 else
9511                         duplex = "half";
9512                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9513                         flow_ctrl = "ON - receive & transmit";
9514                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9515                         flow_ctrl = "ON - transmit";
9516                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9517                         flow_ctrl = "ON - receive";
9518                 else
9519                         flow_ctrl = "none";
9520                 if (bp->link_info.phy_qcfg_resp.option_flags &
9521                     PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9522                         u8 sig_mode = bp->link_info.active_fec_sig_mode &
9523                                       PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9524                         switch (sig_mode) {
9525                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9526                                 signal = "(NRZ) ";
9527                                 break;
9528                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9529                                 signal = "(PAM4) ";
9530                                 break;
9531                         default:
9532                                 break;
9533                         }
9534                 }
9535                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9536                             speed, signal, duplex, flow_ctrl);
9537                 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9538                         netdev_info(bp->dev, "EEE is %s\n",
9539                                     bp->eee.eee_active ? "active" :
9540                                                          "not active");
9541                 fec = bp->link_info.fec_cfg;
9542                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9543                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9544                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9545                                     bnxt_report_fec(&bp->link_info));
9546         } else {
9547                 netif_carrier_off(bp->dev);
9548                 netdev_err(bp->dev, "NIC Link is Down\n");
9549         }
9550 }
9551
9552 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9553 {
9554         if (!resp->supported_speeds_auto_mode &&
9555             !resp->supported_speeds_force_mode &&
9556             !resp->supported_pam4_speeds_auto_mode &&
9557             !resp->supported_pam4_speeds_force_mode)
9558                 return true;
9559         return false;
9560 }
9561
9562 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9563 {
9564         struct bnxt_link_info *link_info = &bp->link_info;
9565         struct hwrm_port_phy_qcaps_output *resp;
9566         struct hwrm_port_phy_qcaps_input *req;
9567         int rc = 0;
9568
9569         if (bp->hwrm_spec_code < 0x10201)
9570                 return 0;
9571
9572         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
9573         if (rc)
9574                 return rc;
9575
9576         resp = hwrm_req_hold(bp, req);
9577         rc = hwrm_req_send(bp, req);
9578         if (rc)
9579                 goto hwrm_phy_qcaps_exit;
9580
9581         bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
9582         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9583                 struct ethtool_eee *eee = &bp->eee;
9584                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9585
9586                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9587                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9588                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9589                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9590                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9591         }
9592
9593         if (bp->hwrm_spec_code >= 0x10a01) {
9594                 if (bnxt_phy_qcaps_no_speed(resp)) {
9595                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9596                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9597                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9598                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9599                         netdev_info(bp->dev, "Ethernet link enabled\n");
9600                         /* Phy re-enabled, reprobe the speeds */
9601                         link_info->support_auto_speeds = 0;
9602                         link_info->support_pam4_auto_speeds = 0;
9603                 }
9604         }
9605         if (resp->supported_speeds_auto_mode)
9606                 link_info->support_auto_speeds =
9607                         le16_to_cpu(resp->supported_speeds_auto_mode);
9608         if (resp->supported_pam4_speeds_auto_mode)
9609                 link_info->support_pam4_auto_speeds =
9610                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9611
9612         bp->port_count = resp->port_cnt;
9613
9614 hwrm_phy_qcaps_exit:
9615         hwrm_req_drop(bp, req);
9616         return rc;
9617 }
9618
9619 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9620 {
9621         u16 diff = advertising ^ supported;
9622
9623         return ((supported | diff) != supported);
9624 }
9625
9626 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9627 {
9628         struct bnxt_link_info *link_info = &bp->link_info;
9629         struct hwrm_port_phy_qcfg_output *resp;
9630         struct hwrm_port_phy_qcfg_input *req;
9631         u8 link_state = link_info->link_state;
9632         bool support_changed = false;
9633         int rc;
9634
9635         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
9636         if (rc)
9637                 return rc;
9638
9639         resp = hwrm_req_hold(bp, req);
9640         rc = hwrm_req_send(bp, req);
9641         if (rc) {
9642                 hwrm_req_drop(bp, req);
9643                 if (BNXT_VF(bp) && rc == -ENODEV) {
9644                         netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
9645                         rc = 0;
9646                 }
9647                 return rc;
9648         }
9649
9650         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9651         link_info->phy_link_status = resp->link;
9652         link_info->duplex = resp->duplex_cfg;
9653         if (bp->hwrm_spec_code >= 0x10800)
9654                 link_info->duplex = resp->duplex_state;
9655         link_info->pause = resp->pause;
9656         link_info->auto_mode = resp->auto_mode;
9657         link_info->auto_pause_setting = resp->auto_pause;
9658         link_info->lp_pause = resp->link_partner_adv_pause;
9659         link_info->force_pause_setting = resp->force_pause;
9660         link_info->duplex_setting = resp->duplex_cfg;
9661         if (link_info->phy_link_status == BNXT_LINK_LINK)
9662                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9663         else
9664                 link_info->link_speed = 0;
9665         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9666         link_info->force_pam4_link_speed =
9667                 le16_to_cpu(resp->force_pam4_link_speed);
9668         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9669         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9670         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9671         link_info->auto_pam4_link_speeds =
9672                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9673         link_info->lp_auto_link_speeds =
9674                 le16_to_cpu(resp->link_partner_adv_speeds);
9675         link_info->lp_auto_pam4_link_speeds =
9676                 resp->link_partner_pam4_adv_speeds;
9677         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9678         link_info->phy_ver[0] = resp->phy_maj;
9679         link_info->phy_ver[1] = resp->phy_min;
9680         link_info->phy_ver[2] = resp->phy_bld;
9681         link_info->media_type = resp->media_type;
9682         link_info->phy_type = resp->phy_type;
9683         link_info->transceiver = resp->xcvr_pkg_type;
9684         link_info->phy_addr = resp->eee_config_phy_addr &
9685                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9686         link_info->module_status = resp->module_status;
9687
9688         if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9689                 struct ethtool_eee *eee = &bp->eee;
9690                 u16 fw_speeds;
9691
9692                 eee->eee_active = 0;
9693                 if (resp->eee_config_phy_addr &
9694                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9695                         eee->eee_active = 1;
9696                         fw_speeds = le16_to_cpu(
9697                                 resp->link_partner_adv_eee_link_speed_mask);
9698                         eee->lp_advertised =
9699                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9700                 }
9701
9702                 /* Pull initial EEE config */
9703                 if (!chng_link_state) {
9704                         if (resp->eee_config_phy_addr &
9705                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9706                                 eee->eee_enabled = 1;
9707
9708                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9709                         eee->advertised =
9710                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9711
9712                         if (resp->eee_config_phy_addr &
9713                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9714                                 __le32 tmr;
9715
9716                                 eee->tx_lpi_enabled = 1;
9717                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9718                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9719                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9720                         }
9721                 }
9722         }
9723
9724         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9725         if (bp->hwrm_spec_code >= 0x10504) {
9726                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9727                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9728         }
9729         /* TODO: need to add more logic to report VF link */
9730         if (chng_link_state) {
9731                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9732                         link_info->link_state = BNXT_LINK_STATE_UP;
9733                 else
9734                         link_info->link_state = BNXT_LINK_STATE_DOWN;
9735                 if (link_state != link_info->link_state)
9736                         bnxt_report_link(bp);
9737         } else {
9738                 /* always link down if not require to update link state */
9739                 link_info->link_state = BNXT_LINK_STATE_DOWN;
9740         }
9741         hwrm_req_drop(bp, req);
9742
9743         if (!BNXT_PHY_CFG_ABLE(bp))
9744                 return 0;
9745
9746         /* Check if any advertised speeds are no longer supported. The caller
9747          * holds the link_lock mutex, so we can modify link_info settings.
9748          */
9749         if (bnxt_support_dropped(link_info->advertising,
9750                                  link_info->support_auto_speeds)) {
9751                 link_info->advertising = link_info->support_auto_speeds;
9752                 support_changed = true;
9753         }
9754         if (bnxt_support_dropped(link_info->advertising_pam4,
9755                                  link_info->support_pam4_auto_speeds)) {
9756                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9757                 support_changed = true;
9758         }
9759         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9760                 bnxt_hwrm_set_link_setting(bp, true, false);
9761         return 0;
9762 }
9763
9764 static void bnxt_get_port_module_status(struct bnxt *bp)
9765 {
9766         struct bnxt_link_info *link_info = &bp->link_info;
9767         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9768         u8 module_status;
9769
9770         if (bnxt_update_link(bp, true))
9771                 return;
9772
9773         module_status = link_info->module_status;
9774         switch (module_status) {
9775         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9776         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9777         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9778                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9779                             bp->pf.port_id);
9780                 if (bp->hwrm_spec_code >= 0x10201) {
9781                         netdev_warn(bp->dev, "Module part number %s\n",
9782                                     resp->phy_vendor_partnumber);
9783                 }
9784                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9785                         netdev_warn(bp->dev, "TX is disabled\n");
9786                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9787                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9788         }
9789 }
9790
9791 static void
9792 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9793 {
9794         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9795                 if (bp->hwrm_spec_code >= 0x10201)
9796                         req->auto_pause =
9797                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9798                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9799                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9800                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9801                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9802                 req->enables |=
9803                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9804         } else {
9805                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9806                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9807                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9808                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9809                 req->enables |=
9810                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9811                 if (bp->hwrm_spec_code >= 0x10201) {
9812                         req->auto_pause = req->force_pause;
9813                         req->enables |= cpu_to_le32(
9814                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9815                 }
9816         }
9817 }
9818
9819 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9820 {
9821         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9822                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9823                 if (bp->link_info.advertising) {
9824                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9825                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9826                 }
9827                 if (bp->link_info.advertising_pam4) {
9828                         req->enables |=
9829                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9830                         req->auto_link_pam4_speed_mask =
9831                                 cpu_to_le16(bp->link_info.advertising_pam4);
9832                 }
9833                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9834                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9835         } else {
9836                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9837                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9838                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9839                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9840                 } else {
9841                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9842                 }
9843         }
9844
9845         /* tell chimp that the setting takes effect immediately */
9846         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9847 }
9848
9849 int bnxt_hwrm_set_pause(struct bnxt *bp)
9850 {
9851         struct hwrm_port_phy_cfg_input *req;
9852         int rc;
9853
9854         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9855         if (rc)
9856                 return rc;
9857
9858         bnxt_hwrm_set_pause_common(bp, req);
9859
9860         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9861             bp->link_info.force_link_chng)
9862                 bnxt_hwrm_set_link_common(bp, req);
9863
9864         rc = hwrm_req_send(bp, req);
9865         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9866                 /* since changing of pause setting doesn't trigger any link
9867                  * change event, the driver needs to update the current pause
9868                  * result upon successfully return of the phy_cfg command
9869                  */
9870                 bp->link_info.pause =
9871                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9872                 bp->link_info.auto_pause_setting = 0;
9873                 if (!bp->link_info.force_link_chng)
9874                         bnxt_report_link(bp);
9875         }
9876         bp->link_info.force_link_chng = false;
9877         return rc;
9878 }
9879
9880 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9881                               struct hwrm_port_phy_cfg_input *req)
9882 {
9883         struct ethtool_eee *eee = &bp->eee;
9884
9885         if (eee->eee_enabled) {
9886                 u16 eee_speeds;
9887                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9888
9889                 if (eee->tx_lpi_enabled)
9890                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9891                 else
9892                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9893
9894                 req->flags |= cpu_to_le32(flags);
9895                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9896                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9897                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9898         } else {
9899                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9900         }
9901 }
9902
9903 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9904 {
9905         struct hwrm_port_phy_cfg_input *req;
9906         int rc;
9907
9908         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9909         if (rc)
9910                 return rc;
9911
9912         if (set_pause)
9913                 bnxt_hwrm_set_pause_common(bp, req);
9914
9915         bnxt_hwrm_set_link_common(bp, req);
9916
9917         if (set_eee)
9918                 bnxt_hwrm_set_eee(bp, req);
9919         return hwrm_req_send(bp, req);
9920 }
9921
9922 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9923 {
9924         struct hwrm_port_phy_cfg_input *req;
9925         int rc;
9926
9927         if (!BNXT_SINGLE_PF(bp))
9928                 return 0;
9929
9930         if (pci_num_vf(bp->pdev) &&
9931             !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9932                 return 0;
9933
9934         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9935         if (rc)
9936                 return rc;
9937
9938         req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9939         rc = hwrm_req_send(bp, req);
9940         if (!rc) {
9941                 mutex_lock(&bp->link_lock);
9942                 /* Device is not obliged link down in certain scenarios, even
9943                  * when forced. Setting the state unknown is consistent with
9944                  * driver startup and will force link state to be reported
9945                  * during subsequent open based on PORT_PHY_QCFG.
9946                  */
9947                 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
9948                 mutex_unlock(&bp->link_lock);
9949         }
9950         return rc;
9951 }
9952
9953 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9954 {
9955 #ifdef CONFIG_TEE_BNXT_FW
9956         int rc = tee_bnxt_fw_load();
9957
9958         if (rc)
9959                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9960
9961         return rc;
9962 #else
9963         netdev_err(bp->dev, "OP-TEE not supported\n");
9964         return -ENODEV;
9965 #endif
9966 }
9967
9968 static int bnxt_try_recover_fw(struct bnxt *bp)
9969 {
9970         if (bp->fw_health && bp->fw_health->status_reliable) {
9971                 int retry = 0, rc;
9972                 u32 sts;
9973
9974                 do {
9975                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9976                         rc = bnxt_hwrm_poll(bp);
9977                         if (!BNXT_FW_IS_BOOTING(sts) &&
9978                             !BNXT_FW_IS_RECOVERING(sts))
9979                                 break;
9980                         retry++;
9981                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9982
9983                 if (!BNXT_FW_IS_HEALTHY(sts)) {
9984                         netdev_err(bp->dev,
9985                                    "Firmware not responding, status: 0x%x\n",
9986                                    sts);
9987                         rc = -ENODEV;
9988                 }
9989                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
9990                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
9991                         return bnxt_fw_reset_via_optee(bp);
9992                 }
9993                 return rc;
9994         }
9995
9996         return -ENODEV;
9997 }
9998
9999 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
10000 {
10001         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10002
10003         if (!BNXT_NEW_RM(bp))
10004                 return; /* no resource reservations required */
10005
10006         hw_resc->resv_cp_rings = 0;
10007         hw_resc->resv_stat_ctxs = 0;
10008         hw_resc->resv_irqs = 0;
10009         hw_resc->resv_tx_rings = 0;
10010         hw_resc->resv_rx_rings = 0;
10011         hw_resc->resv_hw_ring_grps = 0;
10012         hw_resc->resv_vnics = 0;
10013         if (!fw_reset) {
10014                 bp->tx_nr_rings = 0;
10015                 bp->rx_nr_rings = 0;
10016         }
10017 }
10018
10019 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
10020 {
10021         int rc;
10022
10023         if (!BNXT_NEW_RM(bp))
10024                 return 0; /* no resource reservations required */
10025
10026         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
10027         if (rc)
10028                 netdev_err(bp->dev, "resc_qcaps failed\n");
10029
10030         bnxt_clear_reservations(bp, fw_reset);
10031
10032         return rc;
10033 }
10034
10035 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
10036 {
10037         struct hwrm_func_drv_if_change_output *resp;
10038         struct hwrm_func_drv_if_change_input *req;
10039         bool fw_reset = !bp->irq_tbl;
10040         bool resc_reinit = false;
10041         int rc, retry = 0;
10042         u32 flags = 0;
10043
10044         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
10045                 return 0;
10046
10047         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
10048         if (rc)
10049                 return rc;
10050
10051         if (up)
10052                 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
10053         resp = hwrm_req_hold(bp, req);
10054
10055         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
10056         while (retry < BNXT_FW_IF_RETRY) {
10057                 rc = hwrm_req_send(bp, req);
10058                 if (rc != -EAGAIN)
10059                         break;
10060
10061                 msleep(50);
10062                 retry++;
10063         }
10064
10065         if (rc == -EAGAIN) {
10066                 hwrm_req_drop(bp, req);
10067                 return rc;
10068         } else if (!rc) {
10069                 flags = le32_to_cpu(resp->flags);
10070         } else if (up) {
10071                 rc = bnxt_try_recover_fw(bp);
10072                 fw_reset = true;
10073         }
10074         hwrm_req_drop(bp, req);
10075         if (rc)
10076                 return rc;
10077
10078         if (!up) {
10079                 bnxt_inv_fw_health_reg(bp);
10080                 return 0;
10081         }
10082
10083         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
10084                 resc_reinit = true;
10085         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
10086             test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
10087                 fw_reset = true;
10088         else
10089                 bnxt_remap_fw_health_regs(bp);
10090
10091         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
10092                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
10093                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10094                 return -ENODEV;
10095         }
10096         if (resc_reinit || fw_reset) {
10097                 if (fw_reset) {
10098                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10099                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10100                                 bnxt_ulp_stop(bp);
10101                         bnxt_free_ctx_mem(bp);
10102                         kfree(bp->ctx);
10103                         bp->ctx = NULL;
10104                         bnxt_dcb_free(bp);
10105                         rc = bnxt_fw_init_one(bp);
10106                         if (rc) {
10107                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10108                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10109                                 return rc;
10110                         }
10111                         bnxt_clear_int_mode(bp);
10112                         rc = bnxt_init_int_mode(bp);
10113                         if (rc) {
10114                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10115                                 netdev_err(bp->dev, "init int mode failed\n");
10116                                 return rc;
10117                         }
10118                 }
10119                 rc = bnxt_cancel_reservations(bp, fw_reset);
10120         }
10121         return rc;
10122 }
10123
10124 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
10125 {
10126         struct hwrm_port_led_qcaps_output *resp;
10127         struct hwrm_port_led_qcaps_input *req;
10128         struct bnxt_pf_info *pf = &bp->pf;
10129         int rc;
10130
10131         bp->num_leds = 0;
10132         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
10133                 return 0;
10134
10135         rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
10136         if (rc)
10137                 return rc;
10138
10139         req->port_id = cpu_to_le16(pf->port_id);
10140         resp = hwrm_req_hold(bp, req);
10141         rc = hwrm_req_send(bp, req);
10142         if (rc) {
10143                 hwrm_req_drop(bp, req);
10144                 return rc;
10145         }
10146         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
10147                 int i;
10148
10149                 bp->num_leds = resp->num_leds;
10150                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
10151                                                  bp->num_leds);
10152                 for (i = 0; i < bp->num_leds; i++) {
10153                         struct bnxt_led_info *led = &bp->leds[i];
10154                         __le16 caps = led->led_state_caps;
10155
10156                         if (!led->led_group_id ||
10157                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
10158                                 bp->num_leds = 0;
10159                                 break;
10160                         }
10161                 }
10162         }
10163         hwrm_req_drop(bp, req);
10164         return 0;
10165 }
10166
10167 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
10168 {
10169         struct hwrm_wol_filter_alloc_output *resp;
10170         struct hwrm_wol_filter_alloc_input *req;
10171         int rc;
10172
10173         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
10174         if (rc)
10175                 return rc;
10176
10177         req->port_id = cpu_to_le16(bp->pf.port_id);
10178         req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
10179         req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
10180         memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
10181
10182         resp = hwrm_req_hold(bp, req);
10183         rc = hwrm_req_send(bp, req);
10184         if (!rc)
10185                 bp->wol_filter_id = resp->wol_filter_id;
10186         hwrm_req_drop(bp, req);
10187         return rc;
10188 }
10189
10190 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
10191 {
10192         struct hwrm_wol_filter_free_input *req;
10193         int rc;
10194
10195         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
10196         if (rc)
10197                 return rc;
10198
10199         req->port_id = cpu_to_le16(bp->pf.port_id);
10200         req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
10201         req->wol_filter_id = bp->wol_filter_id;
10202
10203         return hwrm_req_send(bp, req);
10204 }
10205
10206 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
10207 {
10208         struct hwrm_wol_filter_qcfg_output *resp;
10209         struct hwrm_wol_filter_qcfg_input *req;
10210         u16 next_handle = 0;
10211         int rc;
10212
10213         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
10214         if (rc)
10215                 return rc;
10216
10217         req->port_id = cpu_to_le16(bp->pf.port_id);
10218         req->handle = cpu_to_le16(handle);
10219         resp = hwrm_req_hold(bp, req);
10220         rc = hwrm_req_send(bp, req);
10221         if (!rc) {
10222                 next_handle = le16_to_cpu(resp->next_handle);
10223                 if (next_handle != 0) {
10224                         if (resp->wol_type ==
10225                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
10226                                 bp->wol = 1;
10227                                 bp->wol_filter_id = resp->wol_filter_id;
10228                         }
10229                 }
10230         }
10231         hwrm_req_drop(bp, req);
10232         return next_handle;
10233 }
10234
10235 static void bnxt_get_wol_settings(struct bnxt *bp)
10236 {
10237         u16 handle = 0;
10238
10239         bp->wol = 0;
10240         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
10241                 return;
10242
10243         do {
10244                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
10245         } while (handle && handle != 0xffff);
10246 }
10247
10248 #ifdef CONFIG_BNXT_HWMON
10249 static ssize_t bnxt_show_temp(struct device *dev,
10250                               struct device_attribute *devattr, char *buf)
10251 {
10252         struct hwrm_temp_monitor_query_output *resp;
10253         struct hwrm_temp_monitor_query_input *req;
10254         struct bnxt *bp = dev_get_drvdata(dev);
10255         u32 len = 0;
10256         int rc;
10257
10258         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10259         if (rc)
10260                 return rc;
10261         resp = hwrm_req_hold(bp, req);
10262         rc = hwrm_req_send(bp, req);
10263         if (!rc)
10264                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
10265         hwrm_req_drop(bp, req);
10266         if (rc)
10267                 return rc;
10268         return len;
10269 }
10270 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
10271
10272 static struct attribute *bnxt_attrs[] = {
10273         &sensor_dev_attr_temp1_input.dev_attr.attr,
10274         NULL
10275 };
10276 ATTRIBUTE_GROUPS(bnxt);
10277
10278 static void bnxt_hwmon_close(struct bnxt *bp)
10279 {
10280         if (bp->hwmon_dev) {
10281                 hwmon_device_unregister(bp->hwmon_dev);
10282                 bp->hwmon_dev = NULL;
10283         }
10284 }
10285
10286 static void bnxt_hwmon_open(struct bnxt *bp)
10287 {
10288         struct hwrm_temp_monitor_query_input *req;
10289         struct pci_dev *pdev = bp->pdev;
10290         int rc;
10291
10292         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10293         if (!rc)
10294                 rc = hwrm_req_send_silent(bp, req);
10295         if (rc == -EACCES || rc == -EOPNOTSUPP) {
10296                 bnxt_hwmon_close(bp);
10297                 return;
10298         }
10299
10300         if (bp->hwmon_dev)
10301                 return;
10302
10303         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
10304                                                           DRV_MODULE_NAME, bp,
10305                                                           bnxt_groups);
10306         if (IS_ERR(bp->hwmon_dev)) {
10307                 bp->hwmon_dev = NULL;
10308                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
10309         }
10310 }
10311 #else
10312 static void bnxt_hwmon_close(struct bnxt *bp)
10313 {
10314 }
10315
10316 static void bnxt_hwmon_open(struct bnxt *bp)
10317 {
10318 }
10319 #endif
10320
10321 static bool bnxt_eee_config_ok(struct bnxt *bp)
10322 {
10323         struct ethtool_eee *eee = &bp->eee;
10324         struct bnxt_link_info *link_info = &bp->link_info;
10325
10326         if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
10327                 return true;
10328
10329         if (eee->eee_enabled) {
10330                 u32 advertising =
10331                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
10332
10333                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10334                         eee->eee_enabled = 0;
10335                         return false;
10336                 }
10337                 if (eee->advertised & ~advertising) {
10338                         eee->advertised = advertising & eee->supported;
10339                         return false;
10340                 }
10341         }
10342         return true;
10343 }
10344
10345 static int bnxt_update_phy_setting(struct bnxt *bp)
10346 {
10347         int rc;
10348         bool update_link = false;
10349         bool update_pause = false;
10350         bool update_eee = false;
10351         struct bnxt_link_info *link_info = &bp->link_info;
10352
10353         rc = bnxt_update_link(bp, true);
10354         if (rc) {
10355                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
10356                            rc);
10357                 return rc;
10358         }
10359         if (!BNXT_SINGLE_PF(bp))
10360                 return 0;
10361
10362         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10363             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
10364             link_info->req_flow_ctrl)
10365                 update_pause = true;
10366         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10367             link_info->force_pause_setting != link_info->req_flow_ctrl)
10368                 update_pause = true;
10369         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10370                 if (BNXT_AUTO_MODE(link_info->auto_mode))
10371                         update_link = true;
10372                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
10373                     link_info->req_link_speed != link_info->force_link_speed)
10374                         update_link = true;
10375                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
10376                          link_info->req_link_speed != link_info->force_pam4_link_speed)
10377                         update_link = true;
10378                 if (link_info->req_duplex != link_info->duplex_setting)
10379                         update_link = true;
10380         } else {
10381                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
10382                         update_link = true;
10383                 if (link_info->advertising != link_info->auto_link_speeds ||
10384                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
10385                         update_link = true;
10386         }
10387
10388         /* The last close may have shutdown the link, so need to call
10389          * PHY_CFG to bring it back up.
10390          */
10391         if (!BNXT_LINK_IS_UP(bp))
10392                 update_link = true;
10393
10394         if (!bnxt_eee_config_ok(bp))
10395                 update_eee = true;
10396
10397         if (update_link)
10398                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
10399         else if (update_pause)
10400                 rc = bnxt_hwrm_set_pause(bp);
10401         if (rc) {
10402                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
10403                            rc);
10404                 return rc;
10405         }
10406
10407         return rc;
10408 }
10409
10410 /* Common routine to pre-map certain register block to different GRC window.
10411  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10412  * in PF and 3 windows in VF that can be customized to map in different
10413  * register blocks.
10414  */
10415 static void bnxt_preset_reg_win(struct bnxt *bp)
10416 {
10417         if (BNXT_PF(bp)) {
10418                 /* CAG registers map to GRC window #4 */
10419                 writel(BNXT_CAG_REG_BASE,
10420                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10421         }
10422 }
10423
10424 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10425
10426 static int bnxt_reinit_after_abort(struct bnxt *bp)
10427 {
10428         int rc;
10429
10430         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10431                 return -EBUSY;
10432
10433         if (bp->dev->reg_state == NETREG_UNREGISTERED)
10434                 return -ENODEV;
10435
10436         rc = bnxt_fw_init_one(bp);
10437         if (!rc) {
10438                 bnxt_clear_int_mode(bp);
10439                 rc = bnxt_init_int_mode(bp);
10440                 if (!rc) {
10441                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10442                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10443                 }
10444         }
10445         return rc;
10446 }
10447
10448 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10449 {
10450         int rc = 0;
10451
10452         bnxt_preset_reg_win(bp);
10453         netif_carrier_off(bp->dev);
10454         if (irq_re_init) {
10455                 /* Reserve rings now if none were reserved at driver probe. */
10456                 rc = bnxt_init_dflt_ring_mode(bp);
10457                 if (rc) {
10458                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10459                         return rc;
10460                 }
10461         }
10462         rc = bnxt_reserve_rings(bp, irq_re_init);
10463         if (rc)
10464                 return rc;
10465         if ((bp->flags & BNXT_FLAG_RFS) &&
10466             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10467                 /* disable RFS if falling back to INTA */
10468                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10469                 bp->flags &= ~BNXT_FLAG_RFS;
10470         }
10471
10472         rc = bnxt_alloc_mem(bp, irq_re_init);
10473         if (rc) {
10474                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10475                 goto open_err_free_mem;
10476         }
10477
10478         if (irq_re_init) {
10479                 bnxt_init_napi(bp);
10480                 rc = bnxt_request_irq(bp);
10481                 if (rc) {
10482                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10483                         goto open_err_irq;
10484                 }
10485         }
10486
10487         rc = bnxt_init_nic(bp, irq_re_init);
10488         if (rc) {
10489                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10490                 goto open_err_irq;
10491         }
10492
10493         bnxt_enable_napi(bp);
10494         bnxt_debug_dev_init(bp);
10495
10496         if (link_re_init) {
10497                 mutex_lock(&bp->link_lock);
10498                 rc = bnxt_update_phy_setting(bp);
10499                 mutex_unlock(&bp->link_lock);
10500                 if (rc) {
10501                         netdev_warn(bp->dev, "failed to update phy settings\n");
10502                         if (BNXT_SINGLE_PF(bp)) {
10503                                 bp->link_info.phy_retry = true;
10504                                 bp->link_info.phy_retry_expires =
10505                                         jiffies + 5 * HZ;
10506                         }
10507                 }
10508         }
10509
10510         if (irq_re_init)
10511                 udp_tunnel_nic_reset_ntf(bp->dev);
10512
10513         if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
10514                 if (!static_key_enabled(&bnxt_xdp_locking_key))
10515                         static_branch_enable(&bnxt_xdp_locking_key);
10516         } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
10517                 static_branch_disable(&bnxt_xdp_locking_key);
10518         }
10519         set_bit(BNXT_STATE_OPEN, &bp->state);
10520         bnxt_enable_int(bp);
10521         /* Enable TX queues */
10522         bnxt_tx_enable(bp);
10523         mod_timer(&bp->timer, jiffies + bp->current_interval);
10524         /* Poll link status and check for SFP+ module status */
10525         mutex_lock(&bp->link_lock);
10526         bnxt_get_port_module_status(bp);
10527         mutex_unlock(&bp->link_lock);
10528
10529         /* VF-reps may need to be re-opened after the PF is re-opened */
10530         if (BNXT_PF(bp))
10531                 bnxt_vf_reps_open(bp);
10532         bnxt_ptp_init_rtc(bp, true);
10533         bnxt_ptp_cfg_tstamp_filters(bp);
10534         return 0;
10535
10536 open_err_irq:
10537         bnxt_del_napi(bp);
10538
10539 open_err_free_mem:
10540         bnxt_free_skbs(bp);
10541         bnxt_free_irq(bp);
10542         bnxt_free_mem(bp, true);
10543         return rc;
10544 }
10545
10546 /* rtnl_lock held */
10547 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10548 {
10549         int rc = 0;
10550
10551         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10552                 rc = -EIO;
10553         if (!rc)
10554                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10555         if (rc) {
10556                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10557                 dev_close(bp->dev);
10558         }
10559         return rc;
10560 }
10561
10562 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10563  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10564  * self tests.
10565  */
10566 int bnxt_half_open_nic(struct bnxt *bp)
10567 {
10568         int rc = 0;
10569
10570         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10571                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
10572                 rc = -ENODEV;
10573                 goto half_open_err;
10574         }
10575
10576         rc = bnxt_alloc_mem(bp, true);
10577         if (rc) {
10578                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10579                 goto half_open_err;
10580         }
10581         set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10582         rc = bnxt_init_nic(bp, true);
10583         if (rc) {
10584                 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10585                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10586                 goto half_open_err;
10587         }
10588         return 0;
10589
10590 half_open_err:
10591         bnxt_free_skbs(bp);
10592         bnxt_free_mem(bp, true);
10593         dev_close(bp->dev);
10594         return rc;
10595 }
10596
10597 /* rtnl_lock held, this call can only be made after a previous successful
10598  * call to bnxt_half_open_nic().
10599  */
10600 void bnxt_half_close_nic(struct bnxt *bp)
10601 {
10602         bnxt_hwrm_resource_free(bp, false, true);
10603         bnxt_free_skbs(bp);
10604         bnxt_free_mem(bp, true);
10605         clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10606 }
10607
10608 void bnxt_reenable_sriov(struct bnxt *bp)
10609 {
10610         if (BNXT_PF(bp)) {
10611                 struct bnxt_pf_info *pf = &bp->pf;
10612                 int n = pf->active_vfs;
10613
10614                 if (n)
10615                         bnxt_cfg_hw_sriov(bp, &n, true);
10616         }
10617 }
10618
10619 static int bnxt_open(struct net_device *dev)
10620 {
10621         struct bnxt *bp = netdev_priv(dev);
10622         int rc;
10623
10624         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10625                 rc = bnxt_reinit_after_abort(bp);
10626                 if (rc) {
10627                         if (rc == -EBUSY)
10628                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10629                         else
10630                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10631                         return -ENODEV;
10632                 }
10633         }
10634
10635         rc = bnxt_hwrm_if_change(bp, true);
10636         if (rc)
10637                 return rc;
10638
10639         rc = __bnxt_open_nic(bp, true, true);
10640         if (rc) {
10641                 bnxt_hwrm_if_change(bp, false);
10642         } else {
10643                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10644                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10645                                 bnxt_ulp_start(bp, 0);
10646                                 bnxt_reenable_sriov(bp);
10647                         }
10648                 }
10649                 bnxt_hwmon_open(bp);
10650         }
10651
10652         return rc;
10653 }
10654
10655 static bool bnxt_drv_busy(struct bnxt *bp)
10656 {
10657         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10658                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
10659 }
10660
10661 static void bnxt_get_ring_stats(struct bnxt *bp,
10662                                 struct rtnl_link_stats64 *stats);
10663
10664 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10665                              bool link_re_init)
10666 {
10667         /* Close the VF-reps before closing PF */
10668         if (BNXT_PF(bp))
10669                 bnxt_vf_reps_close(bp);
10670
10671         /* Change device state to avoid TX queue wake up's */
10672         bnxt_tx_disable(bp);
10673
10674         clear_bit(BNXT_STATE_OPEN, &bp->state);
10675         smp_mb__after_atomic();
10676         while (bnxt_drv_busy(bp))
10677                 msleep(20);
10678
10679         /* Flush rings and disable interrupts */
10680         bnxt_shutdown_nic(bp, irq_re_init);
10681
10682         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10683
10684         bnxt_debug_dev_exit(bp);
10685         bnxt_disable_napi(bp);
10686         del_timer_sync(&bp->timer);
10687         bnxt_free_skbs(bp);
10688
10689         /* Save ring stats before shutdown */
10690         if (bp->bnapi && irq_re_init)
10691                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10692         if (irq_re_init) {
10693                 bnxt_free_irq(bp);
10694                 bnxt_del_napi(bp);
10695         }
10696         bnxt_free_mem(bp, irq_re_init);
10697 }
10698
10699 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10700 {
10701         int rc = 0;
10702
10703         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10704                 /* If we get here, it means firmware reset is in progress
10705                  * while we are trying to close.  We can safely proceed with
10706                  * the close because we are holding rtnl_lock().  Some firmware
10707                  * messages may fail as we proceed to close.  We set the
10708                  * ABORT_ERR flag here so that the FW reset thread will later
10709                  * abort when it gets the rtnl_lock() and sees the flag.
10710                  */
10711                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10712                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10713         }
10714
10715 #ifdef CONFIG_BNXT_SRIOV
10716         if (bp->sriov_cfg) {
10717                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10718                                                       !bp->sriov_cfg,
10719                                                       BNXT_SRIOV_CFG_WAIT_TMO);
10720                 if (rc)
10721                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
10722         }
10723 #endif
10724         __bnxt_close_nic(bp, irq_re_init, link_re_init);
10725         return rc;
10726 }
10727
10728 static int bnxt_close(struct net_device *dev)
10729 {
10730         struct bnxt *bp = netdev_priv(dev);
10731
10732         bnxt_hwmon_close(bp);
10733         bnxt_close_nic(bp, true, true);
10734         bnxt_hwrm_shutdown_link(bp);
10735         bnxt_hwrm_if_change(bp, false);
10736         return 0;
10737 }
10738
10739 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10740                                    u16 *val)
10741 {
10742         struct hwrm_port_phy_mdio_read_output *resp;
10743         struct hwrm_port_phy_mdio_read_input *req;
10744         int rc;
10745
10746         if (bp->hwrm_spec_code < 0x10a00)
10747                 return -EOPNOTSUPP;
10748
10749         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
10750         if (rc)
10751                 return rc;
10752
10753         req->port_id = cpu_to_le16(bp->pf.port_id);
10754         req->phy_addr = phy_addr;
10755         req->reg_addr = cpu_to_le16(reg & 0x1f);
10756         if (mdio_phy_id_is_c45(phy_addr)) {
10757                 req->cl45_mdio = 1;
10758                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10759                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10760                 req->reg_addr = cpu_to_le16(reg);
10761         }
10762
10763         resp = hwrm_req_hold(bp, req);
10764         rc = hwrm_req_send(bp, req);
10765         if (!rc)
10766                 *val = le16_to_cpu(resp->reg_data);
10767         hwrm_req_drop(bp, req);
10768         return rc;
10769 }
10770
10771 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10772                                     u16 val)
10773 {
10774         struct hwrm_port_phy_mdio_write_input *req;
10775         int rc;
10776
10777         if (bp->hwrm_spec_code < 0x10a00)
10778                 return -EOPNOTSUPP;
10779
10780         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
10781         if (rc)
10782                 return rc;
10783
10784         req->port_id = cpu_to_le16(bp->pf.port_id);
10785         req->phy_addr = phy_addr;
10786         req->reg_addr = cpu_to_le16(reg & 0x1f);
10787         if (mdio_phy_id_is_c45(phy_addr)) {
10788                 req->cl45_mdio = 1;
10789                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10790                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10791                 req->reg_addr = cpu_to_le16(reg);
10792         }
10793         req->reg_data = cpu_to_le16(val);
10794
10795         return hwrm_req_send(bp, req);
10796 }
10797
10798 /* rtnl_lock held */
10799 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10800 {
10801         struct mii_ioctl_data *mdio = if_mii(ifr);
10802         struct bnxt *bp = netdev_priv(dev);
10803         int rc;
10804
10805         switch (cmd) {
10806         case SIOCGMIIPHY:
10807                 mdio->phy_id = bp->link_info.phy_addr;
10808
10809                 fallthrough;
10810         case SIOCGMIIREG: {
10811                 u16 mii_regval = 0;
10812
10813                 if (!netif_running(dev))
10814                         return -EAGAIN;
10815
10816                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10817                                              &mii_regval);
10818                 mdio->val_out = mii_regval;
10819                 return rc;
10820         }
10821
10822         case SIOCSMIIREG:
10823                 if (!netif_running(dev))
10824                         return -EAGAIN;
10825
10826                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10827                                                 mdio->val_in);
10828
10829         case SIOCSHWTSTAMP:
10830                 return bnxt_hwtstamp_set(dev, ifr);
10831
10832         case SIOCGHWTSTAMP:
10833                 return bnxt_hwtstamp_get(dev, ifr);
10834
10835         default:
10836                 /* do nothing */
10837                 break;
10838         }
10839         return -EOPNOTSUPP;
10840 }
10841
10842 static void bnxt_get_ring_stats(struct bnxt *bp,
10843                                 struct rtnl_link_stats64 *stats)
10844 {
10845         int i;
10846
10847         for (i = 0; i < bp->cp_nr_rings; i++) {
10848                 struct bnxt_napi *bnapi = bp->bnapi[i];
10849                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10850                 u64 *sw = cpr->stats.sw_stats;
10851
10852                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10853                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10854                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10855
10856                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10857                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10858                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10859
10860                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10861                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10862                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10863
10864                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10865                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10866                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10867
10868                 stats->rx_missed_errors +=
10869                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10870
10871                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10872
10873                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10874
10875                 stats->rx_dropped +=
10876                         cpr->sw_stats.rx.rx_netpoll_discards +
10877                         cpr->sw_stats.rx.rx_oom_discards;
10878         }
10879 }
10880
10881 static void bnxt_add_prev_stats(struct bnxt *bp,
10882                                 struct rtnl_link_stats64 *stats)
10883 {
10884         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10885
10886         stats->rx_packets += prev_stats->rx_packets;
10887         stats->tx_packets += prev_stats->tx_packets;
10888         stats->rx_bytes += prev_stats->rx_bytes;
10889         stats->tx_bytes += prev_stats->tx_bytes;
10890         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10891         stats->multicast += prev_stats->multicast;
10892         stats->rx_dropped += prev_stats->rx_dropped;
10893         stats->tx_dropped += prev_stats->tx_dropped;
10894 }
10895
10896 static void
10897 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10898 {
10899         struct bnxt *bp = netdev_priv(dev);
10900
10901         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10902         /* Make sure bnxt_close_nic() sees that we are reading stats before
10903          * we check the BNXT_STATE_OPEN flag.
10904          */
10905         smp_mb__after_atomic();
10906         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10907                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10908                 *stats = bp->net_stats_prev;
10909                 return;
10910         }
10911
10912         bnxt_get_ring_stats(bp, stats);
10913         bnxt_add_prev_stats(bp, stats);
10914
10915         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10916                 u64 *rx = bp->port_stats.sw_stats;
10917                 u64 *tx = bp->port_stats.sw_stats +
10918                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10919
10920                 stats->rx_crc_errors =
10921                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10922                 stats->rx_frame_errors =
10923                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10924                 stats->rx_length_errors =
10925                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10926                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10927                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10928                 stats->rx_errors =
10929                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10930                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10931                 stats->collisions =
10932                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10933                 stats->tx_fifo_errors =
10934                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10935                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10936         }
10937         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10938 }
10939
10940 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10941 {
10942         struct net_device *dev = bp->dev;
10943         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10944         struct netdev_hw_addr *ha;
10945         u8 *haddr;
10946         int mc_count = 0;
10947         bool update = false;
10948         int off = 0;
10949
10950         netdev_for_each_mc_addr(ha, dev) {
10951                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10952                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10953                         vnic->mc_list_count = 0;
10954                         return false;
10955                 }
10956                 haddr = ha->addr;
10957                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10958                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10959                         update = true;
10960                 }
10961                 off += ETH_ALEN;
10962                 mc_count++;
10963         }
10964         if (mc_count)
10965                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10966
10967         if (mc_count != vnic->mc_list_count) {
10968                 vnic->mc_list_count = mc_count;
10969                 update = true;
10970         }
10971         return update;
10972 }
10973
10974 static bool bnxt_uc_list_updated(struct bnxt *bp)
10975 {
10976         struct net_device *dev = bp->dev;
10977         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10978         struct netdev_hw_addr *ha;
10979         int off = 0;
10980
10981         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10982                 return true;
10983
10984         netdev_for_each_uc_addr(ha, dev) {
10985                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10986                         return true;
10987
10988                 off += ETH_ALEN;
10989         }
10990         return false;
10991 }
10992
10993 static void bnxt_set_rx_mode(struct net_device *dev)
10994 {
10995         struct bnxt *bp = netdev_priv(dev);
10996         struct bnxt_vnic_info *vnic;
10997         bool mc_update = false;
10998         bool uc_update;
10999         u32 mask;
11000
11001         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
11002                 return;
11003
11004         vnic = &bp->vnic_info[0];
11005         mask = vnic->rx_mask;
11006         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
11007                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
11008                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
11009                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
11010
11011         if (dev->flags & IFF_PROMISC)
11012                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11013
11014         uc_update = bnxt_uc_list_updated(bp);
11015
11016         if (dev->flags & IFF_BROADCAST)
11017                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
11018         if (dev->flags & IFF_ALLMULTI) {
11019                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11020                 vnic->mc_list_count = 0;
11021         } else if (dev->flags & IFF_MULTICAST) {
11022                 mc_update = bnxt_mc_list_updated(bp, &mask);
11023         }
11024
11025         if (mask != vnic->rx_mask || uc_update || mc_update) {
11026                 vnic->rx_mask = mask;
11027
11028                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11029                 bnxt_queue_sp_work(bp);
11030         }
11031 }
11032
11033 static int bnxt_cfg_rx_mode(struct bnxt *bp)
11034 {
11035         struct net_device *dev = bp->dev;
11036         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11037         struct hwrm_cfa_l2_filter_free_input *req;
11038         struct netdev_hw_addr *ha;
11039         int i, off = 0, rc;
11040         bool uc_update;
11041
11042         netif_addr_lock_bh(dev);
11043         uc_update = bnxt_uc_list_updated(bp);
11044         netif_addr_unlock_bh(dev);
11045
11046         if (!uc_update)
11047                 goto skip_uc;
11048
11049         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
11050         if (rc)
11051                 return rc;
11052         hwrm_req_hold(bp, req);
11053         for (i = 1; i < vnic->uc_filter_count; i++) {
11054                 req->l2_filter_id = vnic->fw_l2_filter_id[i];
11055
11056                 rc = hwrm_req_send(bp, req);
11057         }
11058         hwrm_req_drop(bp, req);
11059
11060         vnic->uc_filter_count = 1;
11061
11062         netif_addr_lock_bh(dev);
11063         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
11064                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11065         } else {
11066                 netdev_for_each_uc_addr(ha, dev) {
11067                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
11068                         off += ETH_ALEN;
11069                         vnic->uc_filter_count++;
11070                 }
11071         }
11072         netif_addr_unlock_bh(dev);
11073
11074         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
11075                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
11076                 if (rc) {
11077                         if (BNXT_VF(bp) && rc == -ENODEV) {
11078                                 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11079                                         netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
11080                                 else
11081                                         netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
11082                                 rc = 0;
11083                         } else {
11084                                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
11085                         }
11086                         vnic->uc_filter_count = i;
11087                         return rc;
11088                 }
11089         }
11090         if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11091                 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
11092
11093 skip_uc:
11094         if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
11095             !bnxt_promisc_ok(bp))
11096                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11097         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11098         if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
11099                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
11100                             rc);
11101                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11102                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11103                 vnic->mc_list_count = 0;
11104                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11105         }
11106         if (rc)
11107                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
11108                            rc);
11109
11110         return rc;
11111 }
11112
11113 static bool bnxt_can_reserve_rings(struct bnxt *bp)
11114 {
11115 #ifdef CONFIG_BNXT_SRIOV
11116         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
11117                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11118
11119                 /* No minimum rings were provisioned by the PF.  Don't
11120                  * reserve rings by default when device is down.
11121                  */
11122                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
11123                         return true;
11124
11125                 if (!netif_running(bp->dev))
11126                         return false;
11127         }
11128 #endif
11129         return true;
11130 }
11131
11132 /* If the chip and firmware supports RFS */
11133 static bool bnxt_rfs_supported(struct bnxt *bp)
11134 {
11135         if (bp->flags & BNXT_FLAG_CHIP_P5) {
11136                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
11137                         return true;
11138                 return false;
11139         }
11140         /* 212 firmware is broken for aRFS */
11141         if (BNXT_FW_MAJ(bp) == 212)
11142                 return false;
11143         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
11144                 return true;
11145         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11146                 return true;
11147         return false;
11148 }
11149
11150 /* If runtime conditions support RFS */
11151 static bool bnxt_rfs_capable(struct bnxt *bp)
11152 {
11153 #ifdef CONFIG_RFS_ACCEL
11154         int vnics, max_vnics, max_rss_ctxs;
11155
11156         if (bp->flags & BNXT_FLAG_CHIP_P5)
11157                 return bnxt_rfs_supported(bp);
11158         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
11159                 return false;
11160
11161         vnics = 1 + bp->rx_nr_rings;
11162         max_vnics = bnxt_get_max_func_vnics(bp);
11163         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
11164
11165         /* RSS contexts not a limiting factor */
11166         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11167                 max_rss_ctxs = max_vnics;
11168         if (vnics > max_vnics || vnics > max_rss_ctxs) {
11169                 if (bp->rx_nr_rings > 1)
11170                         netdev_warn(bp->dev,
11171                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
11172                                     min(max_rss_ctxs - 1, max_vnics - 1));
11173                 return false;
11174         }
11175
11176         if (!BNXT_NEW_RM(bp))
11177                 return true;
11178
11179         if (vnics == bp->hw_resc.resv_vnics)
11180                 return true;
11181
11182         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
11183         if (vnics <= bp->hw_resc.resv_vnics)
11184                 return true;
11185
11186         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
11187         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
11188         return false;
11189 #else
11190         return false;
11191 #endif
11192 }
11193
11194 static netdev_features_t bnxt_fix_features(struct net_device *dev,
11195                                            netdev_features_t features)
11196 {
11197         struct bnxt *bp = netdev_priv(dev);
11198         netdev_features_t vlan_features;
11199
11200         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
11201                 features &= ~NETIF_F_NTUPLE;
11202
11203         if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
11204                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11205
11206         if (!(features & NETIF_F_GRO))
11207                 features &= ~NETIF_F_GRO_HW;
11208
11209         if (features & NETIF_F_GRO_HW)
11210                 features &= ~NETIF_F_LRO;
11211
11212         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
11213          * turned on or off together.
11214          */
11215         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
11216         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
11217                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11218                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11219                 else if (vlan_features)
11220                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
11221         }
11222 #ifdef CONFIG_BNXT_SRIOV
11223         if (BNXT_VF(bp) && bp->vf.vlan)
11224                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11225 #endif
11226         return features;
11227 }
11228
11229 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
11230 {
11231         struct bnxt *bp = netdev_priv(dev);
11232         u32 flags = bp->flags;
11233         u32 changes;
11234         int rc = 0;
11235         bool re_init = false;
11236         bool update_tpa = false;
11237
11238         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
11239         if (features & NETIF_F_GRO_HW)
11240                 flags |= BNXT_FLAG_GRO;
11241         else if (features & NETIF_F_LRO)
11242                 flags |= BNXT_FLAG_LRO;
11243
11244         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11245                 flags &= ~BNXT_FLAG_TPA;
11246
11247         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11248                 flags |= BNXT_FLAG_STRIP_VLAN;
11249
11250         if (features & NETIF_F_NTUPLE)
11251                 flags |= BNXT_FLAG_RFS;
11252
11253         changes = flags ^ bp->flags;
11254         if (changes & BNXT_FLAG_TPA) {
11255                 update_tpa = true;
11256                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
11257                     (flags & BNXT_FLAG_TPA) == 0 ||
11258                     (bp->flags & BNXT_FLAG_CHIP_P5))
11259                         re_init = true;
11260         }
11261
11262         if (changes & ~BNXT_FLAG_TPA)
11263                 re_init = true;
11264
11265         if (flags != bp->flags) {
11266                 u32 old_flags = bp->flags;
11267
11268                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11269                         bp->flags = flags;
11270                         if (update_tpa)
11271                                 bnxt_set_ring_params(bp);
11272                         return rc;
11273                 }
11274
11275                 if (re_init) {
11276                         bnxt_close_nic(bp, false, false);
11277                         bp->flags = flags;
11278                         if (update_tpa)
11279                                 bnxt_set_ring_params(bp);
11280
11281                         return bnxt_open_nic(bp, false, false);
11282                 }
11283                 if (update_tpa) {
11284                         bp->flags = flags;
11285                         rc = bnxt_set_tpa(bp,
11286                                           (flags & BNXT_FLAG_TPA) ?
11287                                           true : false);
11288                         if (rc)
11289                                 bp->flags = old_flags;
11290                 }
11291         }
11292         return rc;
11293 }
11294
11295 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
11296                               u8 **nextp)
11297 {
11298         struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
11299         struct hop_jumbo_hdr *jhdr;
11300         int hdr_count = 0;
11301         u8 *nexthdr;
11302         int start;
11303
11304         /* Check that there are at most 2 IPv6 extension headers, no
11305          * fragment header, and each is <= 64 bytes.
11306          */
11307         start = nw_off + sizeof(*ip6h);
11308         nexthdr = &ip6h->nexthdr;
11309         while (ipv6_ext_hdr(*nexthdr)) {
11310                 struct ipv6_opt_hdr *hp;
11311                 int hdrlen;
11312
11313                 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
11314                     *nexthdr == NEXTHDR_FRAGMENT)
11315                         return false;
11316                 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
11317                                           skb_headlen(skb), NULL);
11318                 if (!hp)
11319                         return false;
11320                 if (*nexthdr == NEXTHDR_AUTH)
11321                         hdrlen = ipv6_authlen(hp);
11322                 else
11323                         hdrlen = ipv6_optlen(hp);
11324
11325                 if (hdrlen > 64)
11326                         return false;
11327
11328                 /* The ext header may be a hop-by-hop header inserted for
11329                  * big TCP purposes. This will be removed before sending
11330                  * from NIC, so do not count it.
11331                  */
11332                 if (*nexthdr == NEXTHDR_HOP) {
11333                         if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
11334                                 goto increment_hdr;
11335
11336                         jhdr = (struct hop_jumbo_hdr *)hp;
11337                         if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
11338                             jhdr->nexthdr != IPPROTO_TCP)
11339                                 goto increment_hdr;
11340
11341                         goto next_hdr;
11342                 }
11343 increment_hdr:
11344                 hdr_count++;
11345 next_hdr:
11346                 nexthdr = &hp->nexthdr;
11347                 start += hdrlen;
11348         }
11349         if (nextp) {
11350                 /* Caller will check inner protocol */
11351                 if (skb->encapsulation) {
11352                         *nextp = nexthdr;
11353                         return true;
11354                 }
11355                 *nextp = NULL;
11356         }
11357         /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
11358         return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
11359 }
11360
11361 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
11362 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
11363 {
11364         struct udphdr *uh = udp_hdr(skb);
11365         __be16 udp_port = uh->dest;
11366
11367         if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
11368                 return false;
11369         if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
11370                 struct ethhdr *eh = inner_eth_hdr(skb);
11371
11372                 switch (eh->h_proto) {
11373                 case htons(ETH_P_IP):
11374                         return true;
11375                 case htons(ETH_P_IPV6):
11376                         return bnxt_exthdr_check(bp, skb,
11377                                                  skb_inner_network_offset(skb),
11378                                                  NULL);
11379                 }
11380         }
11381         return false;
11382 }
11383
11384 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
11385 {
11386         switch (l4_proto) {
11387         case IPPROTO_UDP:
11388                 return bnxt_udp_tunl_check(bp, skb);
11389         case IPPROTO_IPIP:
11390                 return true;
11391         case IPPROTO_GRE: {
11392                 switch (skb->inner_protocol) {
11393                 default:
11394                         return false;
11395                 case htons(ETH_P_IP):
11396                         return true;
11397                 case htons(ETH_P_IPV6):
11398                         fallthrough;
11399                 }
11400         }
11401         case IPPROTO_IPV6:
11402                 /* Check ext headers of inner ipv6 */
11403                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
11404                                          NULL);
11405         }
11406         return false;
11407 }
11408
11409 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
11410                                              struct net_device *dev,
11411                                              netdev_features_t features)
11412 {
11413         struct bnxt *bp = netdev_priv(dev);
11414         u8 *l4_proto;
11415
11416         features = vlan_features_check(skb, features);
11417         switch (vlan_get_protocol(skb)) {
11418         case htons(ETH_P_IP):
11419                 if (!skb->encapsulation)
11420                         return features;
11421                 l4_proto = &ip_hdr(skb)->protocol;
11422                 if (bnxt_tunl_check(bp, skb, *l4_proto))
11423                         return features;
11424                 break;
11425         case htons(ETH_P_IPV6):
11426                 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
11427                                        &l4_proto))
11428                         break;
11429                 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
11430                         return features;
11431                 break;
11432         }
11433         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11434 }
11435
11436 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
11437                          u32 *reg_buf)
11438 {
11439         struct hwrm_dbg_read_direct_output *resp;
11440         struct hwrm_dbg_read_direct_input *req;
11441         __le32 *dbg_reg_buf;
11442         dma_addr_t mapping;
11443         int rc, i;
11444
11445         rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
11446         if (rc)
11447                 return rc;
11448
11449         dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
11450                                          &mapping);
11451         if (!dbg_reg_buf) {
11452                 rc = -ENOMEM;
11453                 goto dbg_rd_reg_exit;
11454         }
11455
11456         req->host_dest_addr = cpu_to_le64(mapping);
11457
11458         resp = hwrm_req_hold(bp, req);
11459         req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
11460         req->read_len32 = cpu_to_le32(num_words);
11461
11462         rc = hwrm_req_send(bp, req);
11463         if (rc || resp->error_code) {
11464                 rc = -EIO;
11465                 goto dbg_rd_reg_exit;
11466         }
11467         for (i = 0; i < num_words; i++)
11468                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
11469
11470 dbg_rd_reg_exit:
11471         hwrm_req_drop(bp, req);
11472         return rc;
11473 }
11474
11475 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
11476                                        u32 ring_id, u32 *prod, u32 *cons)
11477 {
11478         struct hwrm_dbg_ring_info_get_output *resp;
11479         struct hwrm_dbg_ring_info_get_input *req;
11480         int rc;
11481
11482         rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
11483         if (rc)
11484                 return rc;
11485
11486         req->ring_type = ring_type;
11487         req->fw_ring_id = cpu_to_le32(ring_id);
11488         resp = hwrm_req_hold(bp, req);
11489         rc = hwrm_req_send(bp, req);
11490         if (!rc) {
11491                 *prod = le32_to_cpu(resp->producer_index);
11492                 *cons = le32_to_cpu(resp->consumer_index);
11493         }
11494         hwrm_req_drop(bp, req);
11495         return rc;
11496 }
11497
11498 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11499 {
11500         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11501         int i = bnapi->index;
11502
11503         if (!txr)
11504                 return;
11505
11506         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11507                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11508                     txr->tx_cons);
11509 }
11510
11511 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11512 {
11513         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11514         int i = bnapi->index;
11515
11516         if (!rxr)
11517                 return;
11518
11519         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11520                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11521                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11522                     rxr->rx_sw_agg_prod);
11523 }
11524
11525 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11526 {
11527         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11528         int i = bnapi->index;
11529
11530         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11531                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11532 }
11533
11534 static void bnxt_dbg_dump_states(struct bnxt *bp)
11535 {
11536         int i;
11537         struct bnxt_napi *bnapi;
11538
11539         for (i = 0; i < bp->cp_nr_rings; i++) {
11540                 bnapi = bp->bnapi[i];
11541                 if (netif_msg_drv(bp)) {
11542                         bnxt_dump_tx_sw_state(bnapi);
11543                         bnxt_dump_rx_sw_state(bnapi);
11544                         bnxt_dump_cp_sw_state(bnapi);
11545                 }
11546         }
11547 }
11548
11549 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11550 {
11551         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11552         struct hwrm_ring_reset_input *req;
11553         struct bnxt_napi *bnapi = rxr->bnapi;
11554         struct bnxt_cp_ring_info *cpr;
11555         u16 cp_ring_id;
11556         int rc;
11557
11558         rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
11559         if (rc)
11560                 return rc;
11561
11562         cpr = &bnapi->cp_ring;
11563         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11564         req->cmpl_ring = cpu_to_le16(cp_ring_id);
11565         req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11566         req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11567         return hwrm_req_send_silent(bp, req);
11568 }
11569
11570 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11571 {
11572         if (!silent)
11573                 bnxt_dbg_dump_states(bp);
11574         if (netif_running(bp->dev)) {
11575                 int rc;
11576
11577                 if (silent) {
11578                         bnxt_close_nic(bp, false, false);
11579                         bnxt_open_nic(bp, false, false);
11580                 } else {
11581                         bnxt_ulp_stop(bp);
11582                         bnxt_close_nic(bp, true, false);
11583                         rc = bnxt_open_nic(bp, true, false);
11584                         bnxt_ulp_start(bp, rc);
11585                 }
11586         }
11587 }
11588
11589 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11590 {
11591         struct bnxt *bp = netdev_priv(dev);
11592
11593         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11594         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
11595         bnxt_queue_sp_work(bp);
11596 }
11597
11598 static void bnxt_fw_health_check(struct bnxt *bp)
11599 {
11600         struct bnxt_fw_health *fw_health = bp->fw_health;
11601         u32 val;
11602
11603         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11604                 return;
11605
11606         /* Make sure it is enabled before checking the tmr_counter. */
11607         smp_rmb();
11608         if (fw_health->tmr_counter) {
11609                 fw_health->tmr_counter--;
11610                 return;
11611         }
11612
11613         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11614         if (val == fw_health->last_fw_heartbeat) {
11615                 fw_health->arrests++;
11616                 goto fw_reset;
11617         }
11618
11619         fw_health->last_fw_heartbeat = val;
11620
11621         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11622         if (val != fw_health->last_fw_reset_cnt) {
11623                 fw_health->discoveries++;
11624                 goto fw_reset;
11625         }
11626
11627         fw_health->tmr_counter = fw_health->tmr_multiplier;
11628         return;
11629
11630 fw_reset:
11631         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
11632         bnxt_queue_sp_work(bp);
11633 }
11634
11635 static void bnxt_timer(struct timer_list *t)
11636 {
11637         struct bnxt *bp = from_timer(bp, t, timer);
11638         struct net_device *dev = bp->dev;
11639
11640         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11641                 return;
11642
11643         if (atomic_read(&bp->intr_sem) != 0)
11644                 goto bnxt_restart_timer;
11645
11646         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11647                 bnxt_fw_health_check(bp);
11648
11649         if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) {
11650                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
11651                 bnxt_queue_sp_work(bp);
11652         }
11653
11654         if (bnxt_tc_flower_enabled(bp)) {
11655                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
11656                 bnxt_queue_sp_work(bp);
11657         }
11658
11659 #ifdef CONFIG_RFS_ACCEL
11660         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
11661                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11662                 bnxt_queue_sp_work(bp);
11663         }
11664 #endif /*CONFIG_RFS_ACCEL*/
11665
11666         if (bp->link_info.phy_retry) {
11667                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11668                         bp->link_info.phy_retry = false;
11669                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11670                 } else {
11671                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
11672                         bnxt_queue_sp_work(bp);
11673                 }
11674         }
11675
11676         if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) {
11677                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11678                 bnxt_queue_sp_work(bp);
11679         }
11680
11681         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11682             netif_carrier_ok(dev)) {
11683                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
11684                 bnxt_queue_sp_work(bp);
11685         }
11686 bnxt_restart_timer:
11687         mod_timer(&bp->timer, jiffies + bp->current_interval);
11688 }
11689
11690 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11691 {
11692         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11693          * set.  If the device is being closed, bnxt_close() may be holding
11694          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11695          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11696          */
11697         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11698         rtnl_lock();
11699 }
11700
11701 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11702 {
11703         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11704         rtnl_unlock();
11705 }
11706
11707 /* Only called from bnxt_sp_task() */
11708 static void bnxt_reset(struct bnxt *bp, bool silent)
11709 {
11710         bnxt_rtnl_lock_sp(bp);
11711         if (test_bit(BNXT_STATE_OPEN, &bp->state))
11712                 bnxt_reset_task(bp, silent);
11713         bnxt_rtnl_unlock_sp(bp);
11714 }
11715
11716 /* Only called from bnxt_sp_task() */
11717 static void bnxt_rx_ring_reset(struct bnxt *bp)
11718 {
11719         int i;
11720
11721         bnxt_rtnl_lock_sp(bp);
11722         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11723                 bnxt_rtnl_unlock_sp(bp);
11724                 return;
11725         }
11726         /* Disable and flush TPA before resetting the RX ring */
11727         if (bp->flags & BNXT_FLAG_TPA)
11728                 bnxt_set_tpa(bp, false);
11729         for (i = 0; i < bp->rx_nr_rings; i++) {
11730                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11731                 struct bnxt_cp_ring_info *cpr;
11732                 int rc;
11733
11734                 if (!rxr->bnapi->in_reset)
11735                         continue;
11736
11737                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
11738                 if (rc) {
11739                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
11740                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11741                         else
11742                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11743                                             rc);
11744                         bnxt_reset_task(bp, true);
11745                         break;
11746                 }
11747                 bnxt_free_one_rx_ring_skbs(bp, i);
11748                 rxr->rx_prod = 0;
11749                 rxr->rx_agg_prod = 0;
11750                 rxr->rx_sw_agg_prod = 0;
11751                 rxr->rx_next_cons = 0;
11752                 rxr->bnapi->in_reset = false;
11753                 bnxt_alloc_one_rx_ring(bp, i);
11754                 cpr = &rxr->bnapi->cp_ring;
11755                 cpr->sw_stats.rx.rx_resets++;
11756                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11757                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11758                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11759         }
11760         if (bp->flags & BNXT_FLAG_TPA)
11761                 bnxt_set_tpa(bp, true);
11762         bnxt_rtnl_unlock_sp(bp);
11763 }
11764
11765 static void bnxt_fw_reset_close(struct bnxt *bp)
11766 {
11767         bnxt_ulp_stop(bp);
11768         /* When firmware is in fatal state, quiesce device and disable
11769          * bus master to prevent any potential bad DMAs before freeing
11770          * kernel memory.
11771          */
11772         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11773                 u16 val = 0;
11774
11775                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11776                 if (val == 0xffff)
11777                         bp->fw_reset_min_dsecs = 0;
11778                 bnxt_tx_disable(bp);
11779                 bnxt_disable_napi(bp);
11780                 bnxt_disable_int_sync(bp);
11781                 bnxt_free_irq(bp);
11782                 bnxt_clear_int_mode(bp);
11783                 pci_disable_device(bp->pdev);
11784         }
11785         __bnxt_close_nic(bp, true, false);
11786         bnxt_vf_reps_free(bp);
11787         bnxt_clear_int_mode(bp);
11788         bnxt_hwrm_func_drv_unrgtr(bp);
11789         if (pci_is_enabled(bp->pdev))
11790                 pci_disable_device(bp->pdev);
11791         bnxt_free_ctx_mem(bp);
11792         kfree(bp->ctx);
11793         bp->ctx = NULL;
11794 }
11795
11796 static bool is_bnxt_fw_ok(struct bnxt *bp)
11797 {
11798         struct bnxt_fw_health *fw_health = bp->fw_health;
11799         bool no_heartbeat = false, has_reset = false;
11800         u32 val;
11801
11802         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11803         if (val == fw_health->last_fw_heartbeat)
11804                 no_heartbeat = true;
11805
11806         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11807         if (val != fw_health->last_fw_reset_cnt)
11808                 has_reset = true;
11809
11810         if (!no_heartbeat && has_reset)
11811                 return true;
11812
11813         return false;
11814 }
11815
11816 /* rtnl_lock is acquired before calling this function */
11817 static void bnxt_force_fw_reset(struct bnxt *bp)
11818 {
11819         struct bnxt_fw_health *fw_health = bp->fw_health;
11820         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11821         u32 wait_dsecs;
11822
11823         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11824             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11825                 return;
11826
11827         if (ptp) {
11828                 spin_lock_bh(&ptp->ptp_lock);
11829                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11830                 spin_unlock_bh(&ptp->ptp_lock);
11831         } else {
11832                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11833         }
11834         bnxt_fw_reset_close(bp);
11835         wait_dsecs = fw_health->master_func_wait_dsecs;
11836         if (fw_health->primary) {
11837                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11838                         wait_dsecs = 0;
11839                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11840         } else {
11841                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11842                 wait_dsecs = fw_health->normal_func_wait_dsecs;
11843                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11844         }
11845
11846         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11847         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11848         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11849 }
11850
11851 void bnxt_fw_exception(struct bnxt *bp)
11852 {
11853         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11854         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11855         bnxt_rtnl_lock_sp(bp);
11856         bnxt_force_fw_reset(bp);
11857         bnxt_rtnl_unlock_sp(bp);
11858 }
11859
11860 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11861  * < 0 on error.
11862  */
11863 static int bnxt_get_registered_vfs(struct bnxt *bp)
11864 {
11865 #ifdef CONFIG_BNXT_SRIOV
11866         int rc;
11867
11868         if (!BNXT_PF(bp))
11869                 return 0;
11870
11871         rc = bnxt_hwrm_func_qcfg(bp);
11872         if (rc) {
11873                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11874                 return rc;
11875         }
11876         if (bp->pf.registered_vfs)
11877                 return bp->pf.registered_vfs;
11878         if (bp->sriov_cfg)
11879                 return 1;
11880 #endif
11881         return 0;
11882 }
11883
11884 void bnxt_fw_reset(struct bnxt *bp)
11885 {
11886         bnxt_rtnl_lock_sp(bp);
11887         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11888             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11889                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11890                 int n = 0, tmo;
11891
11892                 if (ptp) {
11893                         spin_lock_bh(&ptp->ptp_lock);
11894                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11895                         spin_unlock_bh(&ptp->ptp_lock);
11896                 } else {
11897                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11898                 }
11899                 if (bp->pf.active_vfs &&
11900                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11901                         n = bnxt_get_registered_vfs(bp);
11902                 if (n < 0) {
11903                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11904                                    n);
11905                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11906                         dev_close(bp->dev);
11907                         goto fw_reset_exit;
11908                 } else if (n > 0) {
11909                         u16 vf_tmo_dsecs = n * 10;
11910
11911                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11912                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11913                         bp->fw_reset_state =
11914                                 BNXT_FW_RESET_STATE_POLL_VF;
11915                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11916                         goto fw_reset_exit;
11917                 }
11918                 bnxt_fw_reset_close(bp);
11919                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11920                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11921                         tmo = HZ / 10;
11922                 } else {
11923                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11924                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11925                 }
11926                 bnxt_queue_fw_reset_work(bp, tmo);
11927         }
11928 fw_reset_exit:
11929         bnxt_rtnl_unlock_sp(bp);
11930 }
11931
11932 static void bnxt_chk_missed_irq(struct bnxt *bp)
11933 {
11934         int i;
11935
11936         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11937                 return;
11938
11939         for (i = 0; i < bp->cp_nr_rings; i++) {
11940                 struct bnxt_napi *bnapi = bp->bnapi[i];
11941                 struct bnxt_cp_ring_info *cpr;
11942                 u32 fw_ring_id;
11943                 int j;
11944
11945                 if (!bnapi)
11946                         continue;
11947
11948                 cpr = &bnapi->cp_ring;
11949                 for (j = 0; j < 2; j++) {
11950                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11951                         u32 val[2];
11952
11953                         if (!cpr2 || cpr2->has_more_work ||
11954                             !bnxt_has_work(bp, cpr2))
11955                                 continue;
11956
11957                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11958                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
11959                                 continue;
11960                         }
11961                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
11962                         bnxt_dbg_hwrm_ring_info_get(bp,
11963                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
11964                                 fw_ring_id, &val[0], &val[1]);
11965                         cpr->sw_stats.cmn.missed_irqs++;
11966                 }
11967         }
11968 }
11969
11970 static void bnxt_cfg_ntp_filters(struct bnxt *);
11971
11972 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
11973 {
11974         struct bnxt_link_info *link_info = &bp->link_info;
11975
11976         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11977                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11978                 if (bp->hwrm_spec_code >= 0x10201) {
11979                         if (link_info->auto_pause_setting &
11980                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11981                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11982                 } else {
11983                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11984                 }
11985                 link_info->advertising = link_info->auto_link_speeds;
11986                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
11987         } else {
11988                 link_info->req_link_speed = link_info->force_link_speed;
11989                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
11990                 if (link_info->force_pam4_link_speed) {
11991                         link_info->req_link_speed =
11992                                 link_info->force_pam4_link_speed;
11993                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
11994                 }
11995                 link_info->req_duplex = link_info->duplex_setting;
11996         }
11997         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
11998                 link_info->req_flow_ctrl =
11999                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
12000         else
12001                 link_info->req_flow_ctrl = link_info->force_pause_setting;
12002 }
12003
12004 static void bnxt_fw_echo_reply(struct bnxt *bp)
12005 {
12006         struct bnxt_fw_health *fw_health = bp->fw_health;
12007         struct hwrm_func_echo_response_input *req;
12008         int rc;
12009
12010         rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
12011         if (rc)
12012                 return;
12013         req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
12014         req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
12015         hwrm_req_send(bp, req);
12016 }
12017
12018 static void bnxt_sp_task(struct work_struct *work)
12019 {
12020         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
12021
12022         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12023         smp_mb__after_atomic();
12024         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12025                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12026                 return;
12027         }
12028
12029         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
12030                 bnxt_cfg_rx_mode(bp);
12031
12032         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
12033                 bnxt_cfg_ntp_filters(bp);
12034         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
12035                 bnxt_hwrm_exec_fwd_req(bp);
12036         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
12037                 bnxt_hwrm_port_qstats(bp, 0);
12038                 bnxt_hwrm_port_qstats_ext(bp, 0);
12039                 bnxt_accumulate_all_stats(bp);
12040         }
12041
12042         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
12043                 int rc;
12044
12045                 mutex_lock(&bp->link_lock);
12046                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
12047                                        &bp->sp_event))
12048                         bnxt_hwrm_phy_qcaps(bp);
12049
12050                 rc = bnxt_update_link(bp, true);
12051                 if (rc)
12052                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
12053                                    rc);
12054
12055                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
12056                                        &bp->sp_event))
12057                         bnxt_init_ethtool_link_settings(bp);
12058                 mutex_unlock(&bp->link_lock);
12059         }
12060         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
12061                 int rc;
12062
12063                 mutex_lock(&bp->link_lock);
12064                 rc = bnxt_update_phy_setting(bp);
12065                 mutex_unlock(&bp->link_lock);
12066                 if (rc) {
12067                         netdev_warn(bp->dev, "update phy settings retry failed\n");
12068                 } else {
12069                         bp->link_info.phy_retry = false;
12070                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
12071                 }
12072         }
12073         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
12074                 mutex_lock(&bp->link_lock);
12075                 bnxt_get_port_module_status(bp);
12076                 mutex_unlock(&bp->link_lock);
12077         }
12078
12079         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
12080                 bnxt_tc_flow_stats_work(bp);
12081
12082         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
12083                 bnxt_chk_missed_irq(bp);
12084
12085         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
12086                 bnxt_fw_echo_reply(bp);
12087
12088         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
12089          * must be the last functions to be called before exiting.
12090          */
12091         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
12092                 bnxt_reset(bp, false);
12093
12094         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
12095                 bnxt_reset(bp, true);
12096
12097         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
12098                 bnxt_rx_ring_reset(bp);
12099
12100         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
12101                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
12102                     test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
12103                         bnxt_devlink_health_fw_report(bp);
12104                 else
12105                         bnxt_fw_reset(bp);
12106         }
12107
12108         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
12109                 if (!is_bnxt_fw_ok(bp))
12110                         bnxt_devlink_health_fw_report(bp);
12111         }
12112
12113         smp_mb__before_atomic();
12114         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12115 }
12116
12117 /* Under rtnl_lock */
12118 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
12119                      int tx_xdp)
12120 {
12121         int max_rx, max_tx, tx_sets = 1;
12122         int tx_rings_needed, stats;
12123         int rx_rings = rx;
12124         int cp, vnics, rc;
12125
12126         if (tcs)
12127                 tx_sets = tcs;
12128
12129         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
12130         if (rc)
12131                 return rc;
12132
12133         if (max_rx < rx)
12134                 return -ENOMEM;
12135
12136         tx_rings_needed = tx * tx_sets + tx_xdp;
12137         if (max_tx < tx_rings_needed)
12138                 return -ENOMEM;
12139
12140         vnics = 1;
12141         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
12142                 vnics += rx_rings;
12143
12144         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12145                 rx_rings <<= 1;
12146         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
12147         stats = cp;
12148         if (BNXT_NEW_RM(bp)) {
12149                 cp += bnxt_get_ulp_msix_num(bp);
12150                 stats += bnxt_get_ulp_stat_ctxs(bp);
12151         }
12152         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
12153                                      stats, vnics);
12154 }
12155
12156 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
12157 {
12158         if (bp->bar2) {
12159                 pci_iounmap(pdev, bp->bar2);
12160                 bp->bar2 = NULL;
12161         }
12162
12163         if (bp->bar1) {
12164                 pci_iounmap(pdev, bp->bar1);
12165                 bp->bar1 = NULL;
12166         }
12167
12168         if (bp->bar0) {
12169                 pci_iounmap(pdev, bp->bar0);
12170                 bp->bar0 = NULL;
12171         }
12172 }
12173
12174 static void bnxt_cleanup_pci(struct bnxt *bp)
12175 {
12176         bnxt_unmap_bars(bp, bp->pdev);
12177         pci_release_regions(bp->pdev);
12178         if (pci_is_enabled(bp->pdev))
12179                 pci_disable_device(bp->pdev);
12180 }
12181
12182 static void bnxt_init_dflt_coal(struct bnxt *bp)
12183 {
12184         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
12185         struct bnxt_coal *coal;
12186         u16 flags = 0;
12187
12188         if (coal_cap->cmpl_params &
12189             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
12190                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
12191
12192         /* Tick values in micro seconds.
12193          * 1 coal_buf x bufs_per_record = 1 completion record.
12194          */
12195         coal = &bp->rx_coal;
12196         coal->coal_ticks = 10;
12197         coal->coal_bufs = 30;
12198         coal->coal_ticks_irq = 1;
12199         coal->coal_bufs_irq = 2;
12200         coal->idle_thresh = 50;
12201         coal->bufs_per_record = 2;
12202         coal->budget = 64;              /* NAPI budget */
12203         coal->flags = flags;
12204
12205         coal = &bp->tx_coal;
12206         coal->coal_ticks = 28;
12207         coal->coal_bufs = 30;
12208         coal->coal_ticks_irq = 2;
12209         coal->coal_bufs_irq = 2;
12210         coal->bufs_per_record = 1;
12211         coal->flags = flags;
12212
12213         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
12214 }
12215
12216 static int bnxt_fw_init_one_p1(struct bnxt *bp)
12217 {
12218         int rc;
12219
12220         bp->fw_cap = 0;
12221         rc = bnxt_hwrm_ver_get(bp);
12222         bnxt_try_map_fw_health_reg(bp);
12223         if (rc) {
12224                 rc = bnxt_try_recover_fw(bp);
12225                 if (rc)
12226                         return rc;
12227                 rc = bnxt_hwrm_ver_get(bp);
12228                 if (rc)
12229                         return rc;
12230         }
12231
12232         bnxt_nvm_cfg_ver_get(bp);
12233
12234         rc = bnxt_hwrm_func_reset(bp);
12235         if (rc)
12236                 return -ENODEV;
12237
12238         bnxt_hwrm_fw_set_time(bp);
12239         return 0;
12240 }
12241
12242 static int bnxt_fw_init_one_p2(struct bnxt *bp)
12243 {
12244         int rc;
12245
12246         /* Get the MAX capabilities for this function */
12247         rc = bnxt_hwrm_func_qcaps(bp);
12248         if (rc) {
12249                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
12250                            rc);
12251                 return -ENODEV;
12252         }
12253
12254         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
12255         if (rc)
12256                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
12257                             rc);
12258
12259         if (bnxt_alloc_fw_health(bp)) {
12260                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
12261         } else {
12262                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
12263                 if (rc)
12264                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
12265                                     rc);
12266         }
12267
12268         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
12269         if (rc)
12270                 return -ENODEV;
12271
12272         bnxt_hwrm_func_qcfg(bp);
12273         bnxt_hwrm_vnic_qcaps(bp);
12274         bnxt_hwrm_port_led_qcaps(bp);
12275         bnxt_ethtool_init(bp);
12276         if (bp->fw_cap & BNXT_FW_CAP_PTP)
12277                 __bnxt_hwrm_ptp_qcfg(bp);
12278         bnxt_dcb_init(bp);
12279         return 0;
12280 }
12281
12282 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
12283 {
12284         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
12285         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
12286                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
12287                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
12288                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
12289         if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
12290                 bp->rss_hash_delta = bp->rss_hash_cfg;
12291         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
12292                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
12293                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
12294                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
12295         }
12296 }
12297
12298 static void bnxt_set_dflt_rfs(struct bnxt *bp)
12299 {
12300         struct net_device *dev = bp->dev;
12301
12302         dev->hw_features &= ~NETIF_F_NTUPLE;
12303         dev->features &= ~NETIF_F_NTUPLE;
12304         bp->flags &= ~BNXT_FLAG_RFS;
12305         if (bnxt_rfs_supported(bp)) {
12306                 dev->hw_features |= NETIF_F_NTUPLE;
12307                 if (bnxt_rfs_capable(bp)) {
12308                         bp->flags |= BNXT_FLAG_RFS;
12309                         dev->features |= NETIF_F_NTUPLE;
12310                 }
12311         }
12312 }
12313
12314 static void bnxt_fw_init_one_p3(struct bnxt *bp)
12315 {
12316         struct pci_dev *pdev = bp->pdev;
12317
12318         bnxt_set_dflt_rss_hash_type(bp);
12319         bnxt_set_dflt_rfs(bp);
12320
12321         bnxt_get_wol_settings(bp);
12322         if (bp->flags & BNXT_FLAG_WOL_CAP)
12323                 device_set_wakeup_enable(&pdev->dev, bp->wol);
12324         else
12325                 device_set_wakeup_capable(&pdev->dev, false);
12326
12327         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
12328         bnxt_hwrm_coal_params_qcaps(bp);
12329 }
12330
12331 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
12332
12333 int bnxt_fw_init_one(struct bnxt *bp)
12334 {
12335         int rc;
12336
12337         rc = bnxt_fw_init_one_p1(bp);
12338         if (rc) {
12339                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
12340                 return rc;
12341         }
12342         rc = bnxt_fw_init_one_p2(bp);
12343         if (rc) {
12344                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
12345                 return rc;
12346         }
12347         rc = bnxt_probe_phy(bp, false);
12348         if (rc)
12349                 return rc;
12350         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
12351         if (rc)
12352                 return rc;
12353
12354         bnxt_fw_init_one_p3(bp);
12355         return 0;
12356 }
12357
12358 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
12359 {
12360         struct bnxt_fw_health *fw_health = bp->fw_health;
12361         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
12362         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
12363         u32 reg_type, reg_off, delay_msecs;
12364
12365         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
12366         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
12367         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
12368         switch (reg_type) {
12369         case BNXT_FW_HEALTH_REG_TYPE_CFG:
12370                 pci_write_config_dword(bp->pdev, reg_off, val);
12371                 break;
12372         case BNXT_FW_HEALTH_REG_TYPE_GRC:
12373                 writel(reg_off & BNXT_GRC_BASE_MASK,
12374                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
12375                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
12376                 fallthrough;
12377         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
12378                 writel(val, bp->bar0 + reg_off);
12379                 break;
12380         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
12381                 writel(val, bp->bar1 + reg_off);
12382                 break;
12383         }
12384         if (delay_msecs) {
12385                 pci_read_config_dword(bp->pdev, 0, &val);
12386                 msleep(delay_msecs);
12387         }
12388 }
12389
12390 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
12391 {
12392         struct hwrm_func_qcfg_output *resp;
12393         struct hwrm_func_qcfg_input *req;
12394         bool result = true; /* firmware will enforce if unknown */
12395
12396         if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
12397                 return result;
12398
12399         if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
12400                 return result;
12401
12402         req->fid = cpu_to_le16(0xffff);
12403         resp = hwrm_req_hold(bp, req);
12404         if (!hwrm_req_send(bp, req))
12405                 result = !!(le16_to_cpu(resp->flags) &
12406                             FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
12407         hwrm_req_drop(bp, req);
12408         return result;
12409 }
12410
12411 static void bnxt_reset_all(struct bnxt *bp)
12412 {
12413         struct bnxt_fw_health *fw_health = bp->fw_health;
12414         int i, rc;
12415
12416         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12417                 bnxt_fw_reset_via_optee(bp);
12418                 bp->fw_reset_timestamp = jiffies;
12419                 return;
12420         }
12421
12422         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
12423                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
12424                         bnxt_fw_reset_writel(bp, i);
12425         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
12426                 struct hwrm_fw_reset_input *req;
12427
12428                 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
12429                 if (!rc) {
12430                         req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
12431                         req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
12432                         req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
12433                         req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
12434                         rc = hwrm_req_send(bp, req);
12435                 }
12436                 if (rc != -ENODEV)
12437                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
12438         }
12439         bp->fw_reset_timestamp = jiffies;
12440 }
12441
12442 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
12443 {
12444         return time_after(jiffies, bp->fw_reset_timestamp +
12445                           (bp->fw_reset_max_dsecs * HZ / 10));
12446 }
12447
12448 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
12449 {
12450         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12451         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
12452                 bnxt_ulp_start(bp, rc);
12453                 bnxt_dl_health_fw_status_update(bp, false);
12454         }
12455         bp->fw_reset_state = 0;
12456         dev_close(bp->dev);
12457 }
12458
12459 static void bnxt_fw_reset_task(struct work_struct *work)
12460 {
12461         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
12462         int rc = 0;
12463
12464         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12465                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
12466                 return;
12467         }
12468
12469         switch (bp->fw_reset_state) {
12470         case BNXT_FW_RESET_STATE_POLL_VF: {
12471                 int n = bnxt_get_registered_vfs(bp);
12472                 int tmo;
12473
12474                 if (n < 0) {
12475                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
12476                                    n, jiffies_to_msecs(jiffies -
12477                                    bp->fw_reset_timestamp));
12478                         goto fw_reset_abort;
12479                 } else if (n > 0) {
12480                         if (bnxt_fw_reset_timeout(bp)) {
12481                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12482                                 bp->fw_reset_state = 0;
12483                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
12484                                            n);
12485                                 return;
12486                         }
12487                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12488                         return;
12489                 }
12490                 bp->fw_reset_timestamp = jiffies;
12491                 rtnl_lock();
12492                 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12493                         bnxt_fw_reset_abort(bp, rc);
12494                         rtnl_unlock();
12495                         return;
12496                 }
12497                 bnxt_fw_reset_close(bp);
12498                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12499                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12500                         tmo = HZ / 10;
12501                 } else {
12502                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12503                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
12504                 }
12505                 rtnl_unlock();
12506                 bnxt_queue_fw_reset_work(bp, tmo);
12507                 return;
12508         }
12509         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
12510                 u32 val;
12511
12512                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12513                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
12514                     !bnxt_fw_reset_timeout(bp)) {
12515                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12516                         return;
12517                 }
12518
12519                 if (!bp->fw_health->primary) {
12520                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
12521
12522                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12523                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12524                         return;
12525                 }
12526                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12527         }
12528                 fallthrough;
12529         case BNXT_FW_RESET_STATE_RESET_FW:
12530                 bnxt_reset_all(bp);
12531                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12532                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
12533                 return;
12534         case BNXT_FW_RESET_STATE_ENABLE_DEV:
12535                 bnxt_inv_fw_health_reg(bp);
12536                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
12537                     !bp->fw_reset_min_dsecs) {
12538                         u16 val;
12539
12540                         pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12541                         if (val == 0xffff) {
12542                                 if (bnxt_fw_reset_timeout(bp)) {
12543                                         netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
12544                                         rc = -ETIMEDOUT;
12545                                         goto fw_reset_abort;
12546                                 }
12547                                 bnxt_queue_fw_reset_work(bp, HZ / 1000);
12548                                 return;
12549                         }
12550                 }
12551                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12552                 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
12553                 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
12554                     !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
12555                         bnxt_dl_remote_reload(bp);
12556                 if (pci_enable_device(bp->pdev)) {
12557                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
12558                         rc = -ENODEV;
12559                         goto fw_reset_abort;
12560                 }
12561                 pci_set_master(bp->pdev);
12562                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12563                 fallthrough;
12564         case BNXT_FW_RESET_STATE_POLL_FW:
12565                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12566                 rc = bnxt_hwrm_poll(bp);
12567                 if (rc) {
12568                         if (bnxt_fw_reset_timeout(bp)) {
12569                                 netdev_err(bp->dev, "Firmware reset aborted\n");
12570                                 goto fw_reset_abort_status;
12571                         }
12572                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12573                         return;
12574                 }
12575                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12576                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12577                 fallthrough;
12578         case BNXT_FW_RESET_STATE_OPENING:
12579                 while (!rtnl_trylock()) {
12580                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12581                         return;
12582                 }
12583                 rc = bnxt_open(bp->dev);
12584                 if (rc) {
12585                         netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
12586                         bnxt_fw_reset_abort(bp, rc);
12587                         rtnl_unlock();
12588                         return;
12589                 }
12590
12591                 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
12592                     bp->fw_health->enabled) {
12593                         bp->fw_health->last_fw_reset_cnt =
12594                                 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12595                 }
12596                 bp->fw_reset_state = 0;
12597                 /* Make sure fw_reset_state is 0 before clearing the flag */
12598                 smp_mb__before_atomic();
12599                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12600                 bnxt_ulp_start(bp, 0);
12601                 bnxt_reenable_sriov(bp);
12602                 bnxt_vf_reps_alloc(bp);
12603                 bnxt_vf_reps_open(bp);
12604                 bnxt_ptp_reapply_pps(bp);
12605                 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
12606                 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
12607                         bnxt_dl_health_fw_recovery_done(bp);
12608                         bnxt_dl_health_fw_status_update(bp, true);
12609                 }
12610                 rtnl_unlock();
12611                 break;
12612         }
12613         return;
12614
12615 fw_reset_abort_status:
12616         if (bp->fw_health->status_reliable ||
12617             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12618                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12619
12620                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12621         }
12622 fw_reset_abort:
12623         rtnl_lock();
12624         bnxt_fw_reset_abort(bp, rc);
12625         rtnl_unlock();
12626 }
12627
12628 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12629 {
12630         int rc;
12631         struct bnxt *bp = netdev_priv(dev);
12632
12633         SET_NETDEV_DEV(dev, &pdev->dev);
12634
12635         /* enable device (incl. PCI PM wakeup), and bus-mastering */
12636         rc = pci_enable_device(pdev);
12637         if (rc) {
12638                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12639                 goto init_err;
12640         }
12641
12642         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12643                 dev_err(&pdev->dev,
12644                         "Cannot find PCI device base address, aborting\n");
12645                 rc = -ENODEV;
12646                 goto init_err_disable;
12647         }
12648
12649         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12650         if (rc) {
12651                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12652                 goto init_err_disable;
12653         }
12654
12655         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12656             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12657                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12658                 rc = -EIO;
12659                 goto init_err_release;
12660         }
12661
12662         pci_set_master(pdev);
12663
12664         bp->dev = dev;
12665         bp->pdev = pdev;
12666
12667         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12668          * determines the BAR size.
12669          */
12670         bp->bar0 = pci_ioremap_bar(pdev, 0);
12671         if (!bp->bar0) {
12672                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12673                 rc = -ENOMEM;
12674                 goto init_err_release;
12675         }
12676
12677         bp->bar2 = pci_ioremap_bar(pdev, 4);
12678         if (!bp->bar2) {
12679                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12680                 rc = -ENOMEM;
12681                 goto init_err_release;
12682         }
12683
12684         INIT_WORK(&bp->sp_task, bnxt_sp_task);
12685         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12686
12687         spin_lock_init(&bp->ntp_fltr_lock);
12688 #if BITS_PER_LONG == 32
12689         spin_lock_init(&bp->db_lock);
12690 #endif
12691
12692         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12693         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12694
12695         timer_setup(&bp->timer, bnxt_timer, 0);
12696         bp->current_interval = BNXT_TIMER_INTERVAL;
12697
12698         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12699         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12700
12701         clear_bit(BNXT_STATE_OPEN, &bp->state);
12702         return 0;
12703
12704 init_err_release:
12705         bnxt_unmap_bars(bp, pdev);
12706         pci_release_regions(pdev);
12707
12708 init_err_disable:
12709         pci_disable_device(pdev);
12710
12711 init_err:
12712         return rc;
12713 }
12714
12715 /* rtnl_lock held */
12716 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12717 {
12718         struct sockaddr *addr = p;
12719         struct bnxt *bp = netdev_priv(dev);
12720         int rc = 0;
12721
12722         if (!is_valid_ether_addr(addr->sa_data))
12723                 return -EADDRNOTAVAIL;
12724
12725         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12726                 return 0;
12727
12728         rc = bnxt_approve_mac(bp, addr->sa_data, true);
12729         if (rc)
12730                 return rc;
12731
12732         eth_hw_addr_set(dev, addr->sa_data);
12733         if (netif_running(dev)) {
12734                 bnxt_close_nic(bp, false, false);
12735                 rc = bnxt_open_nic(bp, false, false);
12736         }
12737
12738         return rc;
12739 }
12740
12741 /* rtnl_lock held */
12742 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12743 {
12744         struct bnxt *bp = netdev_priv(dev);
12745
12746         if (netif_running(dev))
12747                 bnxt_close_nic(bp, true, false);
12748
12749         dev->mtu = new_mtu;
12750         bnxt_set_ring_params(bp);
12751
12752         if (netif_running(dev))
12753                 return bnxt_open_nic(bp, true, false);
12754
12755         return 0;
12756 }
12757
12758 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12759 {
12760         struct bnxt *bp = netdev_priv(dev);
12761         bool sh = false;
12762         int rc;
12763
12764         if (tc > bp->max_tc) {
12765                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12766                            tc, bp->max_tc);
12767                 return -EINVAL;
12768         }
12769
12770         if (netdev_get_num_tc(dev) == tc)
12771                 return 0;
12772
12773         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12774                 sh = true;
12775
12776         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12777                               sh, tc, bp->tx_nr_rings_xdp);
12778         if (rc)
12779                 return rc;
12780
12781         /* Needs to close the device and do hw resource re-allocations */
12782         if (netif_running(bp->dev))
12783                 bnxt_close_nic(bp, true, false);
12784
12785         if (tc) {
12786                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12787                 netdev_set_num_tc(dev, tc);
12788         } else {
12789                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12790                 netdev_reset_tc(dev);
12791         }
12792         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12793         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12794                                bp->tx_nr_rings + bp->rx_nr_rings;
12795
12796         if (netif_running(bp->dev))
12797                 return bnxt_open_nic(bp, true, false);
12798
12799         return 0;
12800 }
12801
12802 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12803                                   void *cb_priv)
12804 {
12805         struct bnxt *bp = cb_priv;
12806
12807         if (!bnxt_tc_flower_enabled(bp) ||
12808             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12809                 return -EOPNOTSUPP;
12810
12811         switch (type) {
12812         case TC_SETUP_CLSFLOWER:
12813                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12814         default:
12815                 return -EOPNOTSUPP;
12816         }
12817 }
12818
12819 LIST_HEAD(bnxt_block_cb_list);
12820
12821 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12822                          void *type_data)
12823 {
12824         struct bnxt *bp = netdev_priv(dev);
12825
12826         switch (type) {
12827         case TC_SETUP_BLOCK:
12828                 return flow_block_cb_setup_simple(type_data,
12829                                                   &bnxt_block_cb_list,
12830                                                   bnxt_setup_tc_block_cb,
12831                                                   bp, bp, true);
12832         case TC_SETUP_QDISC_MQPRIO: {
12833                 struct tc_mqprio_qopt *mqprio = type_data;
12834
12835                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12836
12837                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12838         }
12839         default:
12840                 return -EOPNOTSUPP;
12841         }
12842 }
12843
12844 #ifdef CONFIG_RFS_ACCEL
12845 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12846                             struct bnxt_ntuple_filter *f2)
12847 {
12848         struct flow_keys *keys1 = &f1->fkeys;
12849         struct flow_keys *keys2 = &f2->fkeys;
12850
12851         if (keys1->basic.n_proto != keys2->basic.n_proto ||
12852             keys1->basic.ip_proto != keys2->basic.ip_proto)
12853                 return false;
12854
12855         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12856                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12857                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12858                         return false;
12859         } else {
12860                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12861                            sizeof(keys1->addrs.v6addrs.src)) ||
12862                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12863                            sizeof(keys1->addrs.v6addrs.dst)))
12864                         return false;
12865         }
12866
12867         if (keys1->ports.ports == keys2->ports.ports &&
12868             keys1->control.flags == keys2->control.flags &&
12869             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12870             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12871                 return true;
12872
12873         return false;
12874 }
12875
12876 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12877                               u16 rxq_index, u32 flow_id)
12878 {
12879         struct bnxt *bp = netdev_priv(dev);
12880         struct bnxt_ntuple_filter *fltr, *new_fltr;
12881         struct flow_keys *fkeys;
12882         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12883         int rc = 0, idx, bit_id, l2_idx = 0;
12884         struct hlist_head *head;
12885         u32 flags;
12886
12887         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12888                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12889                 int off = 0, j;
12890
12891                 netif_addr_lock_bh(dev);
12892                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12893                         if (ether_addr_equal(eth->h_dest,
12894                                              vnic->uc_list + off)) {
12895                                 l2_idx = j + 1;
12896                                 break;
12897                         }
12898                 }
12899                 netif_addr_unlock_bh(dev);
12900                 if (!l2_idx)
12901                         return -EINVAL;
12902         }
12903         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12904         if (!new_fltr)
12905                 return -ENOMEM;
12906
12907         fkeys = &new_fltr->fkeys;
12908         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12909                 rc = -EPROTONOSUPPORT;
12910                 goto err_free;
12911         }
12912
12913         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12914              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12915             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12916              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12917                 rc = -EPROTONOSUPPORT;
12918                 goto err_free;
12919         }
12920         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12921             bp->hwrm_spec_code < 0x10601) {
12922                 rc = -EPROTONOSUPPORT;
12923                 goto err_free;
12924         }
12925         flags = fkeys->control.flags;
12926         if (((flags & FLOW_DIS_ENCAPSULATION) &&
12927              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12928                 rc = -EPROTONOSUPPORT;
12929                 goto err_free;
12930         }
12931
12932         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12933         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12934
12935         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12936         head = &bp->ntp_fltr_hash_tbl[idx];
12937         rcu_read_lock();
12938         hlist_for_each_entry_rcu(fltr, head, hash) {
12939                 if (bnxt_fltr_match(fltr, new_fltr)) {
12940                         rc = fltr->sw_id;
12941                         rcu_read_unlock();
12942                         goto err_free;
12943                 }
12944         }
12945         rcu_read_unlock();
12946
12947         spin_lock_bh(&bp->ntp_fltr_lock);
12948         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12949                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
12950         if (bit_id < 0) {
12951                 spin_unlock_bh(&bp->ntp_fltr_lock);
12952                 rc = -ENOMEM;
12953                 goto err_free;
12954         }
12955
12956         new_fltr->sw_id = (u16)bit_id;
12957         new_fltr->flow_id = flow_id;
12958         new_fltr->l2_fltr_idx = l2_idx;
12959         new_fltr->rxq = rxq_index;
12960         hlist_add_head_rcu(&new_fltr->hash, head);
12961         bp->ntp_fltr_count++;
12962         spin_unlock_bh(&bp->ntp_fltr_lock);
12963
12964         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
12965         bnxt_queue_sp_work(bp);
12966
12967         return new_fltr->sw_id;
12968
12969 err_free:
12970         kfree(new_fltr);
12971         return rc;
12972 }
12973
12974 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12975 {
12976         int i;
12977
12978         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
12979                 struct hlist_head *head;
12980                 struct hlist_node *tmp;
12981                 struct bnxt_ntuple_filter *fltr;
12982                 int rc;
12983
12984                 head = &bp->ntp_fltr_hash_tbl[i];
12985                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
12986                         bool del = false;
12987
12988                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
12989                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
12990                                                         fltr->flow_id,
12991                                                         fltr->sw_id)) {
12992                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
12993                                                                          fltr);
12994                                         del = true;
12995                                 }
12996                         } else {
12997                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
12998                                                                        fltr);
12999                                 if (rc)
13000                                         del = true;
13001                                 else
13002                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
13003                         }
13004
13005                         if (del) {
13006                                 spin_lock_bh(&bp->ntp_fltr_lock);
13007                                 hlist_del_rcu(&fltr->hash);
13008                                 bp->ntp_fltr_count--;
13009                                 spin_unlock_bh(&bp->ntp_fltr_lock);
13010                                 synchronize_rcu();
13011                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
13012                                 kfree(fltr);
13013                         }
13014                 }
13015         }
13016         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13017                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13018 }
13019
13020 #else
13021
13022 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13023 {
13024 }
13025
13026 #endif /* CONFIG_RFS_ACCEL */
13027
13028 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
13029 {
13030         struct bnxt *bp = netdev_priv(netdev);
13031         struct udp_tunnel_info ti;
13032         unsigned int cmd;
13033
13034         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
13035         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
13036                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13037         else
13038                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13039
13040         if (ti.port)
13041                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
13042
13043         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
13044 }
13045
13046 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
13047         .sync_table     = bnxt_udp_tunnel_sync,
13048         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
13049                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
13050         .tables         = {
13051                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
13052                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
13053         },
13054 };
13055
13056 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13057                                struct net_device *dev, u32 filter_mask,
13058                                int nlflags)
13059 {
13060         struct bnxt *bp = netdev_priv(dev);
13061
13062         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
13063                                        nlflags, filter_mask, NULL);
13064 }
13065
13066 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
13067                                u16 flags, struct netlink_ext_ack *extack)
13068 {
13069         struct bnxt *bp = netdev_priv(dev);
13070         struct nlattr *attr, *br_spec;
13071         int rem, rc = 0;
13072
13073         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
13074                 return -EOPNOTSUPP;
13075
13076         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13077         if (!br_spec)
13078                 return -EINVAL;
13079
13080         nla_for_each_nested(attr, br_spec, rem) {
13081                 u16 mode;
13082
13083                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
13084                         continue;
13085
13086                 if (nla_len(attr) < sizeof(mode))
13087                         return -EINVAL;
13088
13089                 mode = nla_get_u16(attr);
13090                 if (mode == bp->br_mode)
13091                         break;
13092
13093                 rc = bnxt_hwrm_set_br_mode(bp, mode);
13094                 if (!rc)
13095                         bp->br_mode = mode;
13096                 break;
13097         }
13098         return rc;
13099 }
13100
13101 int bnxt_get_port_parent_id(struct net_device *dev,
13102                             struct netdev_phys_item_id *ppid)
13103 {
13104         struct bnxt *bp = netdev_priv(dev);
13105
13106         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
13107                 return -EOPNOTSUPP;
13108
13109         /* The PF and it's VF-reps only support the switchdev framework */
13110         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
13111                 return -EOPNOTSUPP;
13112
13113         ppid->id_len = sizeof(bp->dsn);
13114         memcpy(ppid->id, bp->dsn, ppid->id_len);
13115
13116         return 0;
13117 }
13118
13119 static const struct net_device_ops bnxt_netdev_ops = {
13120         .ndo_open               = bnxt_open,
13121         .ndo_start_xmit         = bnxt_start_xmit,
13122         .ndo_stop               = bnxt_close,
13123         .ndo_get_stats64        = bnxt_get_stats64,
13124         .ndo_set_rx_mode        = bnxt_set_rx_mode,
13125         .ndo_eth_ioctl          = bnxt_ioctl,
13126         .ndo_validate_addr      = eth_validate_addr,
13127         .ndo_set_mac_address    = bnxt_change_mac_addr,
13128         .ndo_change_mtu         = bnxt_change_mtu,
13129         .ndo_fix_features       = bnxt_fix_features,
13130         .ndo_set_features       = bnxt_set_features,
13131         .ndo_features_check     = bnxt_features_check,
13132         .ndo_tx_timeout         = bnxt_tx_timeout,
13133 #ifdef CONFIG_BNXT_SRIOV
13134         .ndo_get_vf_config      = bnxt_get_vf_config,
13135         .ndo_set_vf_mac         = bnxt_set_vf_mac,
13136         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
13137         .ndo_set_vf_rate        = bnxt_set_vf_bw,
13138         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
13139         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
13140         .ndo_set_vf_trust       = bnxt_set_vf_trust,
13141 #endif
13142         .ndo_setup_tc           = bnxt_setup_tc,
13143 #ifdef CONFIG_RFS_ACCEL
13144         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
13145 #endif
13146         .ndo_bpf                = bnxt_xdp,
13147         .ndo_xdp_xmit           = bnxt_xdp_xmit,
13148         .ndo_bridge_getlink     = bnxt_bridge_getlink,
13149         .ndo_bridge_setlink     = bnxt_bridge_setlink,
13150 };
13151
13152 static void bnxt_remove_one(struct pci_dev *pdev)
13153 {
13154         struct net_device *dev = pci_get_drvdata(pdev);
13155         struct bnxt *bp = netdev_priv(dev);
13156
13157         if (BNXT_PF(bp))
13158                 bnxt_sriov_disable(bp);
13159
13160         bnxt_rdma_aux_device_uninit(bp);
13161
13162         bnxt_ptp_clear(bp);
13163         unregister_netdev(dev);
13164         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13165         /* Flush any pending tasks */
13166         cancel_work_sync(&bp->sp_task);
13167         cancel_delayed_work_sync(&bp->fw_reset_task);
13168         bp->sp_event = 0;
13169
13170         bnxt_dl_fw_reporters_destroy(bp);
13171         bnxt_dl_unregister(bp);
13172         bnxt_shutdown_tc(bp);
13173
13174         bnxt_clear_int_mode(bp);
13175         bnxt_hwrm_func_drv_unrgtr(bp);
13176         bnxt_free_hwrm_resources(bp);
13177         bnxt_ethtool_free(bp);
13178         bnxt_dcb_free(bp);
13179         kfree(bp->ptp_cfg);
13180         bp->ptp_cfg = NULL;
13181         kfree(bp->fw_health);
13182         bp->fw_health = NULL;
13183         bnxt_cleanup_pci(bp);
13184         bnxt_free_ctx_mem(bp);
13185         kfree(bp->ctx);
13186         bp->ctx = NULL;
13187         kfree(bp->rss_indir_tbl);
13188         bp->rss_indir_tbl = NULL;
13189         bnxt_free_port_stats(bp);
13190         free_netdev(dev);
13191 }
13192
13193 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
13194 {
13195         int rc = 0;
13196         struct bnxt_link_info *link_info = &bp->link_info;
13197
13198         bp->phy_flags = 0;
13199         rc = bnxt_hwrm_phy_qcaps(bp);
13200         if (rc) {
13201                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
13202                            rc);
13203                 return rc;
13204         }
13205         if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
13206                 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
13207         else
13208                 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
13209         if (!fw_dflt)
13210                 return 0;
13211
13212         mutex_lock(&bp->link_lock);
13213         rc = bnxt_update_link(bp, false);
13214         if (rc) {
13215                 mutex_unlock(&bp->link_lock);
13216                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
13217                            rc);
13218                 return rc;
13219         }
13220
13221         /* Older firmware does not have supported_auto_speeds, so assume
13222          * that all supported speeds can be autonegotiated.
13223          */
13224         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
13225                 link_info->support_auto_speeds = link_info->support_speeds;
13226
13227         bnxt_init_ethtool_link_settings(bp);
13228         mutex_unlock(&bp->link_lock);
13229         return 0;
13230 }
13231
13232 static int bnxt_get_max_irq(struct pci_dev *pdev)
13233 {
13234         u16 ctrl;
13235
13236         if (!pdev->msix_cap)
13237                 return 1;
13238
13239         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
13240         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
13241 }
13242
13243 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13244                                 int *max_cp)
13245 {
13246         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13247         int max_ring_grps = 0, max_irq;
13248
13249         *max_tx = hw_resc->max_tx_rings;
13250         *max_rx = hw_resc->max_rx_rings;
13251         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
13252         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
13253                         bnxt_get_ulp_msix_num(bp),
13254                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
13255         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
13256                 *max_cp = min_t(int, *max_cp, max_irq);
13257         max_ring_grps = hw_resc->max_hw_ring_grps;
13258         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
13259                 *max_cp -= 1;
13260                 *max_rx -= 2;
13261         }
13262         if (bp->flags & BNXT_FLAG_AGG_RINGS)
13263                 *max_rx >>= 1;
13264         if (bp->flags & BNXT_FLAG_CHIP_P5) {
13265                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
13266                 /* On P5 chips, max_cp output param should be available NQs */
13267                 *max_cp = max_irq;
13268         }
13269         *max_rx = min_t(int, *max_rx, max_ring_grps);
13270 }
13271
13272 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
13273 {
13274         int rx, tx, cp;
13275
13276         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
13277         *max_rx = rx;
13278         *max_tx = tx;
13279         if (!rx || !tx || !cp)
13280                 return -ENOMEM;
13281
13282         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
13283 }
13284
13285 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13286                                bool shared)
13287 {
13288         int rc;
13289
13290         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13291         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
13292                 /* Not enough rings, try disabling agg rings. */
13293                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
13294                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13295                 if (rc) {
13296                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
13297                         bp->flags |= BNXT_FLAG_AGG_RINGS;
13298                         return rc;
13299                 }
13300                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
13301                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13302                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13303                 bnxt_set_ring_params(bp);
13304         }
13305
13306         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
13307                 int max_cp, max_stat, max_irq;
13308
13309                 /* Reserve minimum resources for RoCE */
13310                 max_cp = bnxt_get_max_func_cp_rings(bp);
13311                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
13312                 max_irq = bnxt_get_max_func_irqs(bp);
13313                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
13314                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
13315                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
13316                         return 0;
13317
13318                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
13319                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
13320                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
13321                 max_cp = min_t(int, max_cp, max_irq);
13322                 max_cp = min_t(int, max_cp, max_stat);
13323                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
13324                 if (rc)
13325                         rc = 0;
13326         }
13327         return rc;
13328 }
13329
13330 /* In initial default shared ring setting, each shared ring must have a
13331  * RX/TX ring pair.
13332  */
13333 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
13334 {
13335         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
13336         bp->rx_nr_rings = bp->cp_nr_rings;
13337         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
13338         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13339 }
13340
13341 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
13342 {
13343         int dflt_rings, max_rx_rings, max_tx_rings, rc;
13344
13345         if (!bnxt_can_reserve_rings(bp))
13346                 return 0;
13347
13348         if (sh)
13349                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
13350         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
13351         /* Reduce default rings on multi-port cards so that total default
13352          * rings do not exceed CPU count.
13353          */
13354         if (bp->port_count > 1) {
13355                 int max_rings =
13356                         max_t(int, num_online_cpus() / bp->port_count, 1);
13357
13358                 dflt_rings = min_t(int, dflt_rings, max_rings);
13359         }
13360         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
13361         if (rc)
13362                 return rc;
13363         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
13364         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
13365         if (sh)
13366                 bnxt_trim_dflt_sh_rings(bp);
13367         else
13368                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
13369         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13370
13371         rc = __bnxt_reserve_rings(bp);
13372         if (rc && rc != -ENODEV)
13373                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
13374         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13375         if (sh)
13376                 bnxt_trim_dflt_sh_rings(bp);
13377
13378         /* Rings may have been trimmed, re-reserve the trimmed rings. */
13379         if (bnxt_need_reserve_rings(bp)) {
13380                 rc = __bnxt_reserve_rings(bp);
13381                 if (rc && rc != -ENODEV)
13382                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
13383                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13384         }
13385         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
13386                 bp->rx_nr_rings++;
13387                 bp->cp_nr_rings++;
13388         }
13389         if (rc) {
13390                 bp->tx_nr_rings = 0;
13391                 bp->rx_nr_rings = 0;
13392         }
13393         return rc;
13394 }
13395
13396 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
13397 {
13398         int rc;
13399
13400         if (bp->tx_nr_rings)
13401                 return 0;
13402
13403         bnxt_ulp_irq_stop(bp);
13404         bnxt_clear_int_mode(bp);
13405         rc = bnxt_set_dflt_rings(bp, true);
13406         if (rc) {
13407                 if (BNXT_VF(bp) && rc == -ENODEV)
13408                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13409                 else
13410                         netdev_err(bp->dev, "Not enough rings available.\n");
13411                 goto init_dflt_ring_err;
13412         }
13413         rc = bnxt_init_int_mode(bp);
13414         if (rc)
13415                 goto init_dflt_ring_err;
13416
13417         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13418
13419         bnxt_set_dflt_rfs(bp);
13420
13421 init_dflt_ring_err:
13422         bnxt_ulp_irq_restart(bp, rc);
13423         return rc;
13424 }
13425
13426 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
13427 {
13428         int rc;
13429
13430         ASSERT_RTNL();
13431         bnxt_hwrm_func_qcaps(bp);
13432
13433         if (netif_running(bp->dev))
13434                 __bnxt_close_nic(bp, true, false);
13435
13436         bnxt_ulp_irq_stop(bp);
13437         bnxt_clear_int_mode(bp);
13438         rc = bnxt_init_int_mode(bp);
13439         bnxt_ulp_irq_restart(bp, rc);
13440
13441         if (netif_running(bp->dev)) {
13442                 if (rc)
13443                         dev_close(bp->dev);
13444                 else
13445                         rc = bnxt_open_nic(bp, true, false);
13446         }
13447
13448         return rc;
13449 }
13450
13451 static int bnxt_init_mac_addr(struct bnxt *bp)
13452 {
13453         int rc = 0;
13454
13455         if (BNXT_PF(bp)) {
13456                 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
13457         } else {
13458 #ifdef CONFIG_BNXT_SRIOV
13459                 struct bnxt_vf_info *vf = &bp->vf;
13460                 bool strict_approval = true;
13461
13462                 if (is_valid_ether_addr(vf->mac_addr)) {
13463                         /* overwrite netdev dev_addr with admin VF MAC */
13464                         eth_hw_addr_set(bp->dev, vf->mac_addr);
13465                         /* Older PF driver or firmware may not approve this
13466                          * correctly.
13467                          */
13468                         strict_approval = false;
13469                 } else {
13470                         eth_hw_addr_random(bp->dev);
13471                 }
13472                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
13473 #endif
13474         }
13475         return rc;
13476 }
13477
13478 static void bnxt_vpd_read_info(struct bnxt *bp)
13479 {
13480         struct pci_dev *pdev = bp->pdev;
13481         unsigned int vpd_size, kw_len;
13482         int pos, size;
13483         u8 *vpd_data;
13484
13485         vpd_data = pci_vpd_alloc(pdev, &vpd_size);
13486         if (IS_ERR(vpd_data)) {
13487                 pci_warn(pdev, "Unable to read VPD\n");
13488                 return;
13489         }
13490
13491         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13492                                            PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
13493         if (pos < 0)
13494                 goto read_sn;
13495
13496         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13497         memcpy(bp->board_partno, &vpd_data[pos], size);
13498
13499 read_sn:
13500         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13501                                            PCI_VPD_RO_KEYWORD_SERIALNO,
13502                                            &kw_len);
13503         if (pos < 0)
13504                 goto exit;
13505
13506         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13507         memcpy(bp->board_serialno, &vpd_data[pos], size);
13508 exit:
13509         kfree(vpd_data);
13510 }
13511
13512 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
13513 {
13514         struct pci_dev *pdev = bp->pdev;
13515         u64 qword;
13516
13517         qword = pci_get_dsn(pdev);
13518         if (!qword) {
13519                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
13520                 return -EOPNOTSUPP;
13521         }
13522
13523         put_unaligned_le64(qword, dsn);
13524
13525         bp->flags |= BNXT_FLAG_DSN_VALID;
13526         return 0;
13527 }
13528
13529 static int bnxt_map_db_bar(struct bnxt *bp)
13530 {
13531         if (!bp->db_size)
13532                 return -ENODEV;
13533         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13534         if (!bp->bar1)
13535                 return -ENOMEM;
13536         return 0;
13537 }
13538
13539 void bnxt_print_device_info(struct bnxt *bp)
13540 {
13541         netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
13542                     board_info[bp->board_idx].name,
13543                     (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
13544
13545         pcie_print_link_status(bp->pdev);
13546 }
13547
13548 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13549 {
13550         struct net_device *dev;
13551         struct bnxt *bp;
13552         int rc, max_irqs;
13553
13554         if (pci_is_bridge(pdev))
13555                 return -ENODEV;
13556
13557         /* Clear any pending DMA transactions from crash kernel
13558          * while loading driver in capture kernel.
13559          */
13560         if (is_kdump_kernel()) {
13561                 pci_clear_master(pdev);
13562                 pcie_flr(pdev);
13563         }
13564
13565         max_irqs = bnxt_get_max_irq(pdev);
13566         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13567         if (!dev)
13568                 return -ENOMEM;
13569
13570         bp = netdev_priv(dev);
13571         bp->board_idx = ent->driver_data;
13572         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13573         bnxt_set_max_func_irqs(bp, max_irqs);
13574
13575         if (bnxt_vf_pciid(bp->board_idx))
13576                 bp->flags |= BNXT_FLAG_VF;
13577
13578         /* No devlink port registration in case of a VF */
13579         if (BNXT_PF(bp))
13580                 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
13581
13582         if (pdev->msix_cap)
13583                 bp->flags |= BNXT_FLAG_MSIX_CAP;
13584
13585         rc = bnxt_init_board(pdev, dev);
13586         if (rc < 0)
13587                 goto init_err_free;
13588
13589         dev->netdev_ops = &bnxt_netdev_ops;
13590         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13591         dev->ethtool_ops = &bnxt_ethtool_ops;
13592         pci_set_drvdata(pdev, dev);
13593
13594         rc = bnxt_alloc_hwrm_resources(bp);
13595         if (rc)
13596                 goto init_err_pci_clean;
13597
13598         mutex_init(&bp->hwrm_cmd_lock);
13599         mutex_init(&bp->link_lock);
13600
13601         rc = bnxt_fw_init_one_p1(bp);
13602         if (rc)
13603                 goto init_err_pci_clean;
13604
13605         if (BNXT_PF(bp))
13606                 bnxt_vpd_read_info(bp);
13607
13608         if (BNXT_CHIP_P5(bp)) {
13609                 bp->flags |= BNXT_FLAG_CHIP_P5;
13610                 if (BNXT_CHIP_SR2(bp))
13611                         bp->flags |= BNXT_FLAG_CHIP_SR2;
13612         }
13613
13614         rc = bnxt_alloc_rss_indir_tbl(bp);
13615         if (rc)
13616                 goto init_err_pci_clean;
13617
13618         rc = bnxt_fw_init_one_p2(bp);
13619         if (rc)
13620                 goto init_err_pci_clean;
13621
13622         rc = bnxt_map_db_bar(bp);
13623         if (rc) {
13624                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13625                         rc);
13626                 goto init_err_pci_clean;
13627         }
13628
13629         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13630                            NETIF_F_TSO | NETIF_F_TSO6 |
13631                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13632                            NETIF_F_GSO_IPXIP4 |
13633                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13634                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13635                            NETIF_F_RXCSUM | NETIF_F_GRO;
13636
13637         if (BNXT_SUPPORTS_TPA(bp))
13638                 dev->hw_features |= NETIF_F_LRO;
13639
13640         dev->hw_enc_features =
13641                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13642                         NETIF_F_TSO | NETIF_F_TSO6 |
13643                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13644                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13645                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13646         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13647
13648         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13649                                     NETIF_F_GSO_GRE_CSUM;
13650         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13651         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13652                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13653         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13654                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13655         if (BNXT_SUPPORTS_TPA(bp))
13656                 dev->hw_features |= NETIF_F_GRO_HW;
13657         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13658         if (dev->features & NETIF_F_GRO_HW)
13659                 dev->features &= ~NETIF_F_LRO;
13660         dev->priv_flags |= IFF_UNICAST_FLT;
13661
13662         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
13663
13664         dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
13665                             NETDEV_XDP_ACT_RX_SG;
13666
13667 #ifdef CONFIG_BNXT_SRIOV
13668         init_waitqueue_head(&bp->sriov_cfg_wait);
13669 #endif
13670         if (BNXT_SUPPORTS_TPA(bp)) {
13671                 bp->gro_func = bnxt_gro_func_5730x;
13672                 if (BNXT_CHIP_P4(bp))
13673                         bp->gro_func = bnxt_gro_func_5731x;
13674                 else if (BNXT_CHIP_P5(bp))
13675                         bp->gro_func = bnxt_gro_func_5750x;
13676         }
13677         if (!BNXT_CHIP_P4_PLUS(bp))
13678                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
13679
13680         rc = bnxt_init_mac_addr(bp);
13681         if (rc) {
13682                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13683                 rc = -EADDRNOTAVAIL;
13684                 goto init_err_pci_clean;
13685         }
13686
13687         if (BNXT_PF(bp)) {
13688                 /* Read the adapter's DSN to use as the eswitch switch_id */
13689                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13690         }
13691
13692         /* MTU range: 60 - FW defined max */
13693         dev->min_mtu = ETH_ZLEN;
13694         dev->max_mtu = bp->max_mtu;
13695
13696         rc = bnxt_probe_phy(bp, true);
13697         if (rc)
13698                 goto init_err_pci_clean;
13699
13700         bnxt_set_rx_skb_mode(bp, false);
13701         bnxt_set_tpa_flags(bp);
13702         bnxt_set_ring_params(bp);
13703         rc = bnxt_set_dflt_rings(bp, true);
13704         if (rc) {
13705                 if (BNXT_VF(bp) && rc == -ENODEV) {
13706                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13707                 } else {
13708                         netdev_err(bp->dev, "Not enough rings available.\n");
13709                         rc = -ENOMEM;
13710                 }
13711                 goto init_err_pci_clean;
13712         }
13713
13714         bnxt_fw_init_one_p3(bp);
13715
13716         bnxt_init_dflt_coal(bp);
13717
13718         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13719                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
13720
13721         rc = bnxt_init_int_mode(bp);
13722         if (rc)
13723                 goto init_err_pci_clean;
13724
13725         /* No TC has been set yet and rings may have been trimmed due to
13726          * limited MSIX, so we re-initialize the TX rings per TC.
13727          */
13728         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13729
13730         if (BNXT_PF(bp)) {
13731                 if (!bnxt_pf_wq) {
13732                         bnxt_pf_wq =
13733                                 create_singlethread_workqueue("bnxt_pf_wq");
13734                         if (!bnxt_pf_wq) {
13735                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
13736                                 rc = -ENOMEM;
13737                                 goto init_err_pci_clean;
13738                         }
13739                 }
13740                 rc = bnxt_init_tc(bp);
13741                 if (rc)
13742                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13743                                    rc);
13744         }
13745
13746         bnxt_inv_fw_health_reg(bp);
13747         rc = bnxt_dl_register(bp);
13748         if (rc)
13749                 goto init_err_dl;
13750
13751         rc = register_netdev(dev);
13752         if (rc)
13753                 goto init_err_cleanup;
13754
13755         bnxt_dl_fw_reporters_create(bp);
13756
13757         bnxt_rdma_aux_device_init(bp);
13758
13759         bnxt_print_device_info(bp);
13760
13761         pci_save_state(pdev);
13762
13763         return 0;
13764 init_err_cleanup:
13765         bnxt_dl_unregister(bp);
13766 init_err_dl:
13767         bnxt_shutdown_tc(bp);
13768         bnxt_clear_int_mode(bp);
13769
13770 init_err_pci_clean:
13771         bnxt_hwrm_func_drv_unrgtr(bp);
13772         bnxt_free_hwrm_resources(bp);
13773         bnxt_ethtool_free(bp);
13774         bnxt_ptp_clear(bp);
13775         kfree(bp->ptp_cfg);
13776         bp->ptp_cfg = NULL;
13777         kfree(bp->fw_health);
13778         bp->fw_health = NULL;
13779         bnxt_cleanup_pci(bp);
13780         bnxt_free_ctx_mem(bp);
13781         kfree(bp->ctx);
13782         bp->ctx = NULL;
13783         kfree(bp->rss_indir_tbl);
13784         bp->rss_indir_tbl = NULL;
13785
13786 init_err_free:
13787         free_netdev(dev);
13788         return rc;
13789 }
13790
13791 static void bnxt_shutdown(struct pci_dev *pdev)
13792 {
13793         struct net_device *dev = pci_get_drvdata(pdev);
13794         struct bnxt *bp;
13795
13796         if (!dev)
13797                 return;
13798
13799         rtnl_lock();
13800         bp = netdev_priv(dev);
13801         if (!bp)
13802                 goto shutdown_exit;
13803
13804         if (netif_running(dev))
13805                 dev_close(dev);
13806
13807         bnxt_clear_int_mode(bp);
13808         pci_disable_device(pdev);
13809
13810         if (system_state == SYSTEM_POWER_OFF) {
13811                 pci_wake_from_d3(pdev, bp->wol);
13812                 pci_set_power_state(pdev, PCI_D3hot);
13813         }
13814
13815 shutdown_exit:
13816         rtnl_unlock();
13817 }
13818
13819 #ifdef CONFIG_PM_SLEEP
13820 static int bnxt_suspend(struct device *device)
13821 {
13822         struct net_device *dev = dev_get_drvdata(device);
13823         struct bnxt *bp = netdev_priv(dev);
13824         int rc = 0;
13825
13826         rtnl_lock();
13827         bnxt_ulp_stop(bp);
13828         if (netif_running(dev)) {
13829                 netif_device_detach(dev);
13830                 rc = bnxt_close(dev);
13831         }
13832         bnxt_hwrm_func_drv_unrgtr(bp);
13833         pci_disable_device(bp->pdev);
13834         bnxt_free_ctx_mem(bp);
13835         kfree(bp->ctx);
13836         bp->ctx = NULL;
13837         rtnl_unlock();
13838         return rc;
13839 }
13840
13841 static int bnxt_resume(struct device *device)
13842 {
13843         struct net_device *dev = dev_get_drvdata(device);
13844         struct bnxt *bp = netdev_priv(dev);
13845         int rc = 0;
13846
13847         rtnl_lock();
13848         rc = pci_enable_device(bp->pdev);
13849         if (rc) {
13850                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13851                            rc);
13852                 goto resume_exit;
13853         }
13854         pci_set_master(bp->pdev);
13855         if (bnxt_hwrm_ver_get(bp)) {
13856                 rc = -ENODEV;
13857                 goto resume_exit;
13858         }
13859         rc = bnxt_hwrm_func_reset(bp);
13860         if (rc) {
13861                 rc = -EBUSY;
13862                 goto resume_exit;
13863         }
13864
13865         rc = bnxt_hwrm_func_qcaps(bp);
13866         if (rc)
13867                 goto resume_exit;
13868
13869         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13870                 rc = -ENODEV;
13871                 goto resume_exit;
13872         }
13873
13874         bnxt_get_wol_settings(bp);
13875         if (netif_running(dev)) {
13876                 rc = bnxt_open(dev);
13877                 if (!rc)
13878                         netif_device_attach(dev);
13879         }
13880
13881 resume_exit:
13882         bnxt_ulp_start(bp, rc);
13883         if (!rc)
13884                 bnxt_reenable_sriov(bp);
13885         rtnl_unlock();
13886         return rc;
13887 }
13888
13889 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13890 #define BNXT_PM_OPS (&bnxt_pm_ops)
13891
13892 #else
13893
13894 #define BNXT_PM_OPS NULL
13895
13896 #endif /* CONFIG_PM_SLEEP */
13897
13898 /**
13899  * bnxt_io_error_detected - called when PCI error is detected
13900  * @pdev: Pointer to PCI device
13901  * @state: The current pci connection state
13902  *
13903  * This function is called after a PCI bus error affecting
13904  * this device has been detected.
13905  */
13906 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13907                                                pci_channel_state_t state)
13908 {
13909         struct net_device *netdev = pci_get_drvdata(pdev);
13910         struct bnxt *bp = netdev_priv(netdev);
13911
13912         netdev_info(netdev, "PCI I/O error detected\n");
13913
13914         rtnl_lock();
13915         netif_device_detach(netdev);
13916
13917         bnxt_ulp_stop(bp);
13918
13919         if (state == pci_channel_io_perm_failure) {
13920                 rtnl_unlock();
13921                 return PCI_ERS_RESULT_DISCONNECT;
13922         }
13923
13924         if (state == pci_channel_io_frozen)
13925                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13926
13927         if (netif_running(netdev))
13928                 bnxt_close(netdev);
13929
13930         if (pci_is_enabled(pdev))
13931                 pci_disable_device(pdev);
13932         bnxt_free_ctx_mem(bp);
13933         kfree(bp->ctx);
13934         bp->ctx = NULL;
13935         rtnl_unlock();
13936
13937         /* Request a slot slot reset. */
13938         return PCI_ERS_RESULT_NEED_RESET;
13939 }
13940
13941 /**
13942  * bnxt_io_slot_reset - called after the pci bus has been reset.
13943  * @pdev: Pointer to PCI device
13944  *
13945  * Restart the card from scratch, as if from a cold-boot.
13946  * At this point, the card has exprienced a hard reset,
13947  * followed by fixups by BIOS, and has its config space
13948  * set up identically to what it was at cold boot.
13949  */
13950 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
13951 {
13952         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
13953         struct net_device *netdev = pci_get_drvdata(pdev);
13954         struct bnxt *bp = netdev_priv(netdev);
13955         int retry = 0;
13956         int err = 0;
13957         int off;
13958
13959         netdev_info(bp->dev, "PCI Slot Reset\n");
13960
13961         rtnl_lock();
13962
13963         if (pci_enable_device(pdev)) {
13964                 dev_err(&pdev->dev,
13965                         "Cannot re-enable PCI device after reset.\n");
13966         } else {
13967                 pci_set_master(pdev);
13968                 /* Upon fatal error, our device internal logic that latches to
13969                  * BAR value is getting reset and will restore only upon
13970                  * rewritting the BARs.
13971                  *
13972                  * As pci_restore_state() does not re-write the BARs if the
13973                  * value is same as saved value earlier, driver needs to
13974                  * write the BARs to 0 to force restore, in case of fatal error.
13975                  */
13976                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
13977                                        &bp->state)) {
13978                         for (off = PCI_BASE_ADDRESS_0;
13979                              off <= PCI_BASE_ADDRESS_5; off += 4)
13980                                 pci_write_config_dword(bp->pdev, off, 0);
13981                 }
13982                 pci_restore_state(pdev);
13983                 pci_save_state(pdev);
13984
13985                 bnxt_inv_fw_health_reg(bp);
13986                 bnxt_try_map_fw_health_reg(bp);
13987
13988                 /* In some PCIe AER scenarios, firmware may take up to
13989                  * 10 seconds to become ready in the worst case.
13990                  */
13991                 do {
13992                         err = bnxt_try_recover_fw(bp);
13993                         if (!err)
13994                                 break;
13995                         retry++;
13996                 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
13997
13998                 if (err) {
13999                         dev_err(&pdev->dev, "Firmware not ready\n");
14000                         goto reset_exit;
14001                 }
14002
14003                 err = bnxt_hwrm_func_reset(bp);
14004                 if (!err)
14005                         result = PCI_ERS_RESULT_RECOVERED;
14006
14007                 bnxt_ulp_irq_stop(bp);
14008                 bnxt_clear_int_mode(bp);
14009                 err = bnxt_init_int_mode(bp);
14010                 bnxt_ulp_irq_restart(bp, err);
14011         }
14012
14013 reset_exit:
14014         bnxt_clear_reservations(bp, true);
14015         rtnl_unlock();
14016
14017         return result;
14018 }
14019
14020 /**
14021  * bnxt_io_resume - called when traffic can start flowing again.
14022  * @pdev: Pointer to PCI device
14023  *
14024  * This callback is called when the error recovery driver tells
14025  * us that its OK to resume normal operation.
14026  */
14027 static void bnxt_io_resume(struct pci_dev *pdev)
14028 {
14029         struct net_device *netdev = pci_get_drvdata(pdev);
14030         struct bnxt *bp = netdev_priv(netdev);
14031         int err;
14032
14033         netdev_info(bp->dev, "PCI Slot Resume\n");
14034         rtnl_lock();
14035
14036         err = bnxt_hwrm_func_qcaps(bp);
14037         if (!err && netif_running(netdev))
14038                 err = bnxt_open(netdev);
14039
14040         bnxt_ulp_start(bp, err);
14041         if (!err) {
14042                 bnxt_reenable_sriov(bp);
14043                 netif_device_attach(netdev);
14044         }
14045
14046         rtnl_unlock();
14047 }
14048
14049 static const struct pci_error_handlers bnxt_err_handler = {
14050         .error_detected = bnxt_io_error_detected,
14051         .slot_reset     = bnxt_io_slot_reset,
14052         .resume         = bnxt_io_resume
14053 };
14054
14055 static struct pci_driver bnxt_pci_driver = {
14056         .name           = DRV_MODULE_NAME,
14057         .id_table       = bnxt_pci_tbl,
14058         .probe          = bnxt_init_one,
14059         .remove         = bnxt_remove_one,
14060         .shutdown       = bnxt_shutdown,
14061         .driver.pm      = BNXT_PM_OPS,
14062         .err_handler    = &bnxt_err_handler,
14063 #if defined(CONFIG_BNXT_SRIOV)
14064         .sriov_configure = bnxt_sriov_configure,
14065 #endif
14066 };
14067
14068 static int __init bnxt_init(void)
14069 {
14070         int err;
14071
14072         bnxt_debug_init();
14073         err = pci_register_driver(&bnxt_pci_driver);
14074         if (err) {
14075                 bnxt_debug_exit();
14076                 return err;
14077         }
14078
14079         return 0;
14080 }
14081
14082 static void __exit bnxt_exit(void)
14083 {
14084         pci_unregister_driver(&bnxt_pci_driver);
14085         if (bnxt_pf_wq)
14086                 destroy_workqueue(bnxt_pf_wq);
14087         bnxt_debug_exit();
14088 }
14089
14090 module_init(bnxt_init);
14091 module_exit(bnxt_exit);