drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/of_net.h>
   6 #include <linux/pci.h>
   7 #include <linux/bpf.h>
   8 #include <generated/utsrelease.h>
   9 #include <linux/crash_dump.h>
  10
  11 /* Local includes */
  12 #include "i40e.h"
  13 #include "i40e_diag.h"
  14 #include "i40e_xsk.h"
  15 #include <net/udp_tunnel.h>
  16 #include <net/xdp_sock_drv.h>
  17 /* All i40e tracepoints are defined by the include below, which
  18  * must be included exactly once across the whole kernel with
  19  * CREATE_TRACE_POINTS defined
  20  */
  21 #define CREATE_TRACE_POINTS
  22 #include "i40e_trace.h"
  23
  24 const char i40e_driver_name[] = "i40e";
  25 static const char i40e_driver_string[] =
  26                         "Intel(R) Ethernet Connection XL710 Network Driver";
  27
  28 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
  29
  30 /* a bit of forward declarations */
  31 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  32 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  33 static int i40e_add_vsi(struct i40e_vsi *vsi);
  34 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  35 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  36 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  37 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  38 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  39 static void i40e_prep_for_reset(struct i40e_pf *pf);
  40 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
  41                                    bool lock_acquired);
  42 static int i40e_reset(struct i40e_pf *pf);
  43 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  44 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
  45 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
  46 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
  47 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
  48 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  49 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  50 static int i40e_get_capabilities(struct i40e_pf *pf,
  51                                  enum i40e_admin_queue_opc list_type);
  52 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
  53
  54 /* i40e_pci_tbl - PCI Device ID Table
  55  *
  56  * Last entry must be all 0s
  57  *
  58  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  59  *   Class, Class Mask, private data (not used) }
  60  */
  61 static const struct pci_device_id i40e_pci_tbl[] = {
  62         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  63         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  64         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  86         /* required last entry */
  87         {0, }
  88 };
  89 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  90
  91 #define I40E_MAX_VF_COUNT 128
  92 static int debug = -1;
  93 module_param(debug, uint, 0);
  94 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
  95
  96 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  97 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  98 MODULE_LICENSE("GPL v2");
  99
 100 static struct workqueue_struct *i40e_wq;
 101
 102 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
 103                                   struct net_device *netdev, int delta)
 104 {
 105         struct netdev_hw_addr *ha;
 106
 107         if (!f || !netdev)
 108                 return;
 109
 110         netdev_for_each_mc_addr(ha, netdev) {
 111                 if (ether_addr_equal(ha->addr, f->macaddr)) {
 112                         ha->refcount += delta;
 113                         if (ha->refcount <= 0)
 114                                 ha->refcount = 1;
 115                         break;
 116                 }
 117         }
 118 }
 119
 120 /**
 121  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 122  * @hw:   pointer to the HW structure
 123  * @mem:  ptr to mem struct to fill out
 124  * @size: size of memory requested
 125  * @alignment: what to align the allocation to
 126  **/
 127 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 128                             u64 size, u32 alignment)
 129 {
 130         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 131
 132         mem->size = ALIGN(size, alignment);
 133         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
 134                                      GFP_KERNEL);
 135         if (!mem->va)
 136                 return -ENOMEM;
 137
 138         return 0;
 139 }
 140
 141 /**
 142  * i40e_free_dma_mem_d - OS specific memory free for shared code
 143  * @hw:   pointer to the HW structure
 144  * @mem:  ptr to mem struct to free
 145  **/
 146 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 147 {
 148         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 149
 150         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 151         mem->va = NULL;
 152         mem->pa = 0;
 153         mem->size = 0;
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 160  * @hw:   pointer to the HW structure
 161  * @mem:  ptr to mem struct to fill out
 162  * @size: size of memory requested
 163  **/
 164 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 165                              u32 size)
 166 {
 167         mem->size = size;
 168         mem->va = kzalloc(size, GFP_KERNEL);
 169
 170         if (!mem->va)
 171                 return -ENOMEM;
 172
 173         return 0;
 174 }
 175
 176 /**
 177  * i40e_free_virt_mem_d - OS specific memory free for shared code
 178  * @hw:   pointer to the HW structure
 179  * @mem:  ptr to mem struct to free
 180  **/
 181 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 182 {
 183         /* it's ok to kfree a NULL pointer */
 184         kfree(mem->va);
 185         mem->va = NULL;
 186         mem->size = 0;
 187
 188         return 0;
 189 }
 190
 191 /**
 192  * i40e_get_lump - find a lump of free generic resource
 193  * @pf: board private structure
 194  * @pile: the pile of resource to search
 195  * @needed: the number of items needed
 196  * @id: an owner id to stick on the items assigned
 197  *
 198  * Returns the base item index of the lump, or negative for error
 199  *
 200  * The search_hint trick and lack of advanced fit-finding only work
 201  * because we're highly likely to have all the same size lump requests.
 202  * Linear search time and any fragmentation should be minimal.
 203  **/
 204 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 205                          u16 needed, u16 id)
 206 {
 207         int ret = -ENOMEM;
 208         int i, j;
 209
 210         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 211                 dev_info(&pf->pdev->dev,
 212                          "param err: pile=%s needed=%d id=0x%04x\n",
 213                          pile ? "<valid>" : "<null>", needed, id);
 214                 return -EINVAL;
 215         }
 216
 217         /* start the linear search with an imperfect hint */
 218         i = pile->search_hint;
 219         while (i < pile->num_entries) {
 220                 /* skip already allocated entries */
 221                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 222                         i++;
 223                         continue;
 224                 }
 225
 226                 /* do we have enough in this lump? */
 227                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 228                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 229                                 break;
 230                 }
 231
 232                 if (j == needed) {
 233                         /* there was enough, so assign it to the requestor */
 234                         for (j = 0; j < needed; j++)
 235                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 236                         ret = i;
 237                         pile->search_hint = i + j;
 238                         break;
 239                 }
 240
 241                 /* not enough, so skip over it and continue looking */
 242                 i += j;
 243         }
 244
 245         return ret;
 246 }
 247
 248 /**
 249  * i40e_put_lump - return a lump of generic resource
 250  * @pile: the pile of resource to search
 251  * @index: the base item index
 252  * @id: the owner id of the items assigned
 253  *
 254  * Returns the count of items in the lump
 255  **/
 256 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 257 {
 258         int valid_id = (id | I40E_PILE_VALID_BIT);
 259         int count = 0;
 260         int i;
 261
 262         if (!pile || index >= pile->num_entries)
 263                 return -EINVAL;
 264
 265         for (i = index;
 266              i < pile->num_entries && pile->list[i] == valid_id;
 267              i++) {
 268                 pile->list[i] = 0;
 269                 count++;
 270         }
 271
 272         if (count && index < pile->search_hint)
 273                 pile->search_hint = index;
 274
 275         return count;
 276 }
 277
 278 /**
 279  * i40e_find_vsi_from_id - searches for the vsi with the given id
 280  * @pf: the pf structure to search for the vsi
 281  * @id: id of the vsi it is searching for
 282  **/
 283 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 284 {
 285         int i;
 286
 287         for (i = 0; i < pf->num_alloc_vsi; i++)
 288                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 289                         return pf->vsi[i];
 290
 291         return NULL;
 292 }
 293
 294 /**
 295  * i40e_service_event_schedule - Schedule the service task to wake up
 296  * @pf: board private structure
 297  *
 298  * If not already scheduled, this puts the task into the work queue
 299  **/
 300 void i40e_service_event_schedule(struct i40e_pf *pf)
 301 {
 302         if ((!test_bit(__I40E_DOWN, pf->state) &&
 303              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
 304               test_bit(__I40E_RECOVERY_MODE, pf->state))
 305                 queue_work(i40e_wq, &pf->service_task);
 306 }
 307
 308 /**
 309  * i40e_tx_timeout - Respond to a Tx Hang
 310  * @netdev: network interface device structure
 311  * @txqueue: queue number timing out
 312  *
 313  * If any port has noticed a Tx timeout, it is likely that the whole
 314  * device is munged, not just the one netdev port, so go for the full
 315  * reset.
 316  **/
 317 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 318 {
 319         struct i40e_netdev_priv *np = netdev_priv(netdev);
 320         struct i40e_vsi *vsi = np->vsi;
 321         struct i40e_pf *pf = vsi->back;
 322         struct i40e_ring *tx_ring = NULL;
 323         unsigned int i;
 324         u32 head, val;
 325
 326         pf->tx_timeout_count++;
 327
 328         /* with txqueue index, find the tx_ring struct */
 329         for (i = 0; i < vsi->num_queue_pairs; i++) {
 330                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 331                         if (txqueue ==
 332                             vsi->tx_rings[i]->queue_index) {
 333                                 tx_ring = vsi->tx_rings[i];
 334                                 break;
 335                         }
 336                 }
 337         }
 338
 339         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 340                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 341         else if (time_before(jiffies,
 342                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 343                 return;   /* don't do any new action before the next timeout */
 344
 345         /* don't kick off another recovery if one is already pending */
 346         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 347                 return;
 348
 349         if (tx_ring) {
 350                 head = i40e_get_head(tx_ring);
 351                 /* Read interrupt register */
 352                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 353                         val = rd32(&pf->hw,
 354                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 355                                                 tx_ring->vsi->base_vector - 1));
 356                 else
 357                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 358
 359                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 360                             vsi->seid, txqueue, tx_ring->next_to_clean,
 361                             head, tx_ring->next_to_use,
 362                             readl(tx_ring->tail), val);
 363         }
 364
 365         pf->tx_timeout_last_recovery = jiffies;
 366         netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
 367                     pf->tx_timeout_recovery_level, txqueue);
 368
 369         switch (pf->tx_timeout_recovery_level) {
 370         case 1:
 371                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 372                 break;
 373         case 2:
 374                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 375                 break;
 376         case 3:
 377                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 378                 break;
 379         default:
 380                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 381                 break;
 382         }
 383
 384         i40e_service_event_schedule(pf);
 385         pf->tx_timeout_recovery_level++;
 386 }
 387
 388 /**
 389  * i40e_get_vsi_stats_struct - Get System Network Statistics
 390  * @vsi: the VSI we care about
 391  *
 392  * Returns the address of the device statistics structure.
 393  * The statistics are actually updated from the service task.
 394  **/
 395 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 396 {
 397         return &vsi->net_stats;
 398 }
 399
 400 /**
 401  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 402  * @ring: Tx ring to get statistics from
 403  * @stats: statistics entry to be updated
 404  **/
 405 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 406                                             struct rtnl_link_stats64 *stats)
 407 {
 408         u64 bytes, packets;
 409         unsigned int start;
 410
 411         do {
 412                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 413                 packets = ring->stats.packets;
 414                 bytes   = ring->stats.bytes;
 415         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 416
 417         stats->tx_packets += packets;
 418         stats->tx_bytes   += bytes;
 419 }
 420
 421 /**
 422  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 423  * @netdev: network interface device structure
 424  * @stats: data structure to store statistics
 425  *
 426  * Returns the address of the device statistics structure.
 427  * The statistics are actually updated from the service task.
 428  **/
 429 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 430                                   struct rtnl_link_stats64 *stats)
 431 {
 432         struct i40e_netdev_priv *np = netdev_priv(netdev);
 433         struct i40e_vsi *vsi = np->vsi;
 434         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 435         struct i40e_ring *ring;
 436         int i;
 437
 438         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 439                 return;
 440
 441         if (!vsi->tx_rings)
 442                 return;
 443
 444         rcu_read_lock();
 445         for (i = 0; i < vsi->num_queue_pairs; i++) {
 446                 u64 bytes, packets;
 447                 unsigned int start;
 448
 449                 ring = READ_ONCE(vsi->tx_rings[i]);
 450                 if (!ring)
 451                         continue;
 452                 i40e_get_netdev_stats_struct_tx(ring, stats);
 453
 454                 if (i40e_enabled_xdp_vsi(vsi)) {
 455                         ring = READ_ONCE(vsi->xdp_rings[i]);
 456                         if (!ring)
 457                                 continue;
 458                         i40e_get_netdev_stats_struct_tx(ring, stats);
 459                 }
 460
 461                 ring = READ_ONCE(vsi->rx_rings[i]);
 462                 if (!ring)
 463                         continue;
 464                 do {
 465                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
 466                         packets = ring->stats.packets;
 467                         bytes   = ring->stats.bytes;
 468                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 469
 470                 stats->rx_packets += packets;
 471                 stats->rx_bytes   += bytes;
 472
 473         }
 474         rcu_read_unlock();
 475
 476         /* following stats updated by i40e_watchdog_subtask() */
 477         stats->multicast        = vsi_stats->multicast;
 478         stats->tx_errors        = vsi_stats->tx_errors;
 479         stats->tx_dropped       = vsi_stats->tx_dropped;
 480         stats->rx_errors        = vsi_stats->rx_errors;
 481         stats->rx_dropped       = vsi_stats->rx_dropped;
 482         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 483         stats->rx_length_errors = vsi_stats->rx_length_errors;
 484 }
 485
 486 /**
 487  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 488  * @vsi: the VSI to have its stats reset
 489  **/
 490 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 491 {
 492         struct rtnl_link_stats64 *ns;
 493         int i;
 494
 495         if (!vsi)
 496                 return;
 497
 498         ns = i40e_get_vsi_stats_struct(vsi);
 499         memset(ns, 0, sizeof(*ns));
 500         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 501         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 502         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 503         if (vsi->rx_rings && vsi->rx_rings[0]) {
 504                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 505                         memset(&vsi->rx_rings[i]->stats, 0,
 506                                sizeof(vsi->rx_rings[i]->stats));
 507                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 508                                sizeof(vsi->rx_rings[i]->rx_stats));
 509                         memset(&vsi->tx_rings[i]->stats, 0,
 510                                sizeof(vsi->tx_rings[i]->stats));
 511                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 512                                sizeof(vsi->tx_rings[i]->tx_stats));
 513                 }
 514         }
 515         vsi->stat_offsets_loaded = false;
 516 }
 517
 518 /**
 519  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 520  * @pf: the PF to be reset
 521  **/
 522 void i40e_pf_reset_stats(struct i40e_pf *pf)
 523 {
 524         int i;
 525
 526         memset(&pf->stats, 0, sizeof(pf->stats));
 527         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 528         pf->stat_offsets_loaded = false;
 529
 530         for (i = 0; i < I40E_MAX_VEB; i++) {
 531                 if (pf->veb[i]) {
 532                         memset(&pf->veb[i]->stats, 0,
 533                                sizeof(pf->veb[i]->stats));
 534                         memset(&pf->veb[i]->stats_offsets, 0,
 535                                sizeof(pf->veb[i]->stats_offsets));
 536                         memset(&pf->veb[i]->tc_stats, 0,
 537                                sizeof(pf->veb[i]->tc_stats));
 538                         memset(&pf->veb[i]->tc_stats_offsets, 0,
 539                                sizeof(pf->veb[i]->tc_stats_offsets));
 540                         pf->veb[i]->stat_offsets_loaded = false;
 541                 }
 542         }
 543         pf->hw_csum_rx_error = 0;
 544 }
 545
 546 /**
 547  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 548  * @hw: ptr to the hardware info
 549  * @hireg: the high 32 bit reg to read
 550  * @loreg: the low 32 bit reg to read
 551  * @offset_loaded: has the initial offset been loaded yet
 552  * @offset: ptr to current offset value
 553  * @stat: ptr to the stat
 554  *
 555  * Since the device stats are not reset at PFReset, they likely will not
 556  * be zeroed when the driver starts.  We'll save the first values read
 557  * and use them as offsets to be subtracted from the raw values in order
 558  * to report stats that count from zero.  In the process, we also manage
 559  * the potential roll-over.
 560  **/
 561 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 562                                bool offset_loaded, u64 *offset, u64 *stat)
 563 {
 564         u64 new_data;
 565
 566         if (hw->device_id == I40E_DEV_ID_QEMU) {
 567                 new_data = rd32(hw, loreg);
 568                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 569         } else {
 570                 new_data = rd64(hw, loreg);
 571         }
 572         if (!offset_loaded)
 573                 *offset = new_data;
 574         if (likely(new_data >= *offset))
 575                 *stat = new_data - *offset;
 576         else
 577                 *stat = (new_data + BIT_ULL(48)) - *offset;
 578         *stat &= 0xFFFFFFFFFFFFULL;
 579 }
 580
 581 /**
 582  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 583  * @hw: ptr to the hardware info
 584  * @reg: the hw reg to read
 585  * @offset_loaded: has the initial offset been loaded yet
 586  * @offset: ptr to current offset value
 587  * @stat: ptr to the stat
 588  **/
 589 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 590                                bool offset_loaded, u64 *offset, u64 *stat)
 591 {
 592         u32 new_data;
 593
 594         new_data = rd32(hw, reg);
 595         if (!offset_loaded)
 596                 *offset = new_data;
 597         if (likely(new_data >= *offset))
 598                 *stat = (u32)(new_data - *offset);
 599         else
 600                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 601 }
 602
 603 /**
 604  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 605  * @hw: ptr to the hardware info
 606  * @reg: the hw reg to read and clear
 607  * @stat: ptr to the stat
 608  **/
 609 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 610 {
 611         u32 new_data = rd32(hw, reg);
 612
 613         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 614         *stat += new_data;
 615 }
 616
 617 /**
 618  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 619  * @vsi: the VSI to be updated
 620  **/
 621 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 622 {
 623         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 624         struct i40e_pf *pf = vsi->back;
 625         struct i40e_hw *hw = &pf->hw;
 626         struct i40e_eth_stats *oes;
 627         struct i40e_eth_stats *es;     /* device's eth stats */
 628
 629         es = &vsi->eth_stats;
 630         oes = &vsi->eth_stats_offsets;
 631
 632         /* Gather up the stats that the hw collects */
 633         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 634                            vsi->stat_offsets_loaded,
 635                            &oes->tx_errors, &es->tx_errors);
 636         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 637                            vsi->stat_offsets_loaded,
 638                            &oes->rx_discards, &es->rx_discards);
 639         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 640                            vsi->stat_offsets_loaded,
 641                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 642
 643         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 644                            I40E_GLV_GORCL(stat_idx),
 645                            vsi->stat_offsets_loaded,
 646                            &oes->rx_bytes, &es->rx_bytes);
 647         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 648                            I40E_GLV_UPRCL(stat_idx),
 649                            vsi->stat_offsets_loaded,
 650                            &oes->rx_unicast, &es->rx_unicast);
 651         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 652                            I40E_GLV_MPRCL(stat_idx),
 653                            vsi->stat_offsets_loaded,
 654                            &oes->rx_multicast, &es->rx_multicast);
 655         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 656                            I40E_GLV_BPRCL(stat_idx),
 657                            vsi->stat_offsets_loaded,
 658                            &oes->rx_broadcast, &es->rx_broadcast);
 659
 660         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 661                            I40E_GLV_GOTCL(stat_idx),
 662                            vsi->stat_offsets_loaded,
 663                            &oes->tx_bytes, &es->tx_bytes);
 664         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 665                            I40E_GLV_UPTCL(stat_idx),
 666                            vsi->stat_offsets_loaded,
 667                            &oes->tx_unicast, &es->tx_unicast);
 668         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 669                            I40E_GLV_MPTCL(stat_idx),
 670                            vsi->stat_offsets_loaded,
 671                            &oes->tx_multicast, &es->tx_multicast);
 672         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 673                            I40E_GLV_BPTCL(stat_idx),
 674                            vsi->stat_offsets_loaded,
 675                            &oes->tx_broadcast, &es->tx_broadcast);
 676         vsi->stat_offsets_loaded = true;
 677 }
 678
 679 /**
 680  * i40e_update_veb_stats - Update Switch component statistics
 681  * @veb: the VEB being updated
 682  **/
 683 void i40e_update_veb_stats(struct i40e_veb *veb)
 684 {
 685         struct i40e_pf *pf = veb->pf;
 686         struct i40e_hw *hw = &pf->hw;
 687         struct i40e_eth_stats *oes;
 688         struct i40e_eth_stats *es;     /* device's eth stats */
 689         struct i40e_veb_tc_stats *veb_oes;
 690         struct i40e_veb_tc_stats *veb_es;
 691         int i, idx = 0;
 692
 693         idx = veb->stats_idx;
 694         es = &veb->stats;
 695         oes = &veb->stats_offsets;
 696         veb_es = &veb->tc_stats;
 697         veb_oes = &veb->tc_stats_offsets;
 698
 699         /* Gather up the stats that the hw collects */
 700         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 701                            veb->stat_offsets_loaded,
 702                            &oes->tx_discards, &es->tx_discards);
 703         if (hw->revision_id > 0)
 704                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 705                                    veb->stat_offsets_loaded,
 706                                    &oes->rx_unknown_protocol,
 707                                    &es->rx_unknown_protocol);
 708         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 709                            veb->stat_offsets_loaded,
 710                            &oes->rx_bytes, &es->rx_bytes);
 711         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 712                            veb->stat_offsets_loaded,
 713                            &oes->rx_unicast, &es->rx_unicast);
 714         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 715                            veb->stat_offsets_loaded,
 716                            &oes->rx_multicast, &es->rx_multicast);
 717         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 718                            veb->stat_offsets_loaded,
 719                            &oes->rx_broadcast, &es->rx_broadcast);
 720
 721         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 722                            veb->stat_offsets_loaded,
 723                            &oes->tx_bytes, &es->tx_bytes);
 724         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 725                            veb->stat_offsets_loaded,
 726                            &oes->tx_unicast, &es->tx_unicast);
 727         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 728                            veb->stat_offsets_loaded,
 729                            &oes->tx_multicast, &es->tx_multicast);
 730         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 731                            veb->stat_offsets_loaded,
 732                            &oes->tx_broadcast, &es->tx_broadcast);
 733         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 734                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 735                                    I40E_GLVEBTC_RPCL(i, idx),
 736                                    veb->stat_offsets_loaded,
 737                                    &veb_oes->tc_rx_packets[i],
 738                                    &veb_es->tc_rx_packets[i]);
 739                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 740                                    I40E_GLVEBTC_RBCL(i, idx),
 741                                    veb->stat_offsets_loaded,
 742                                    &veb_oes->tc_rx_bytes[i],
 743                                    &veb_es->tc_rx_bytes[i]);
 744                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 745                                    I40E_GLVEBTC_TPCL(i, idx),
 746                                    veb->stat_offsets_loaded,
 747                                    &veb_oes->tc_tx_packets[i],
 748                                    &veb_es->tc_tx_packets[i]);
 749                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 750                                    I40E_GLVEBTC_TBCL(i, idx),
 751                                    veb->stat_offsets_loaded,
 752                                    &veb_oes->tc_tx_bytes[i],
 753                                    &veb_es->tc_tx_bytes[i]);
 754         }
 755         veb->stat_offsets_loaded = true;
 756 }
 757
 758 /**
 759  * i40e_update_vsi_stats - Update the vsi statistics counters.
 760  * @vsi: the VSI to be updated
 761  *
 762  * There are a few instances where we store the same stat in a
 763  * couple of different structs.  This is partly because we have
 764  * the netdev stats that need to be filled out, which is slightly
 765  * different from the "eth_stats" defined by the chip and used in
 766  * VF communications.  We sort it out here.
 767  **/
 768 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 769 {
 770         struct i40e_pf *pf = vsi->back;
 771         struct rtnl_link_stats64 *ons;
 772         struct rtnl_link_stats64 *ns;   /* netdev stats */
 773         struct i40e_eth_stats *oes;
 774         struct i40e_eth_stats *es;     /* device's eth stats */
 775         u32 tx_restart, tx_busy;
 776         struct i40e_ring *p;
 777         u32 rx_page, rx_buf;
 778         u64 bytes, packets;
 779         unsigned int start;
 780         u64 tx_linearize;
 781         u64 tx_force_wb;
 782         u64 rx_p, rx_b;
 783         u64 tx_p, tx_b;
 784         u16 q;
 785
 786         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 787             test_bit(__I40E_CONFIG_BUSY, pf->state))
 788                 return;
 789
 790         ns = i40e_get_vsi_stats_struct(vsi);
 791         ons = &vsi->net_stats_offsets;
 792         es = &vsi->eth_stats;
 793         oes = &vsi->eth_stats_offsets;
 794
 795         /* Gather up the netdev and vsi stats that the driver collects
 796          * on the fly during packet processing
 797          */
 798         rx_b = rx_p = 0;
 799         tx_b = tx_p = 0;
 800         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 801         rx_page = 0;
 802         rx_buf = 0;
 803         rcu_read_lock();
 804         for (q = 0; q < vsi->num_queue_pairs; q++) {
 805                 /* locate Tx ring */
 806                 p = READ_ONCE(vsi->tx_rings[q]);
 807                 if (!p)
 808                         continue;
 809
 810                 do {
 811                         start = u64_stats_fetch_begin_irq(&p->syncp);
 812                         packets = p->stats.packets;
 813                         bytes = p->stats.bytes;
 814                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 815                 tx_b += bytes;
 816                 tx_p += packets;
 817                 tx_restart += p->tx_stats.restart_queue;
 818                 tx_busy += p->tx_stats.tx_busy;
 819                 tx_linearize += p->tx_stats.tx_linearize;
 820                 tx_force_wb += p->tx_stats.tx_force_wb;
 821
 822                 /* locate Rx ring */
 823                 p = READ_ONCE(vsi->rx_rings[q]);
 824                 if (!p)
 825                         continue;
 826
 827                 do {
 828                         start = u64_stats_fetch_begin_irq(&p->syncp);
 829                         packets = p->stats.packets;
 830                         bytes = p->stats.bytes;
 831                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 832                 rx_b += bytes;
 833                 rx_p += packets;
 834                 rx_buf += p->rx_stats.alloc_buff_failed;
 835                 rx_page += p->rx_stats.alloc_page_failed;
 836
 837                 if (i40e_enabled_xdp_vsi(vsi)) {
 838                         /* locate XDP ring */
 839                         p = READ_ONCE(vsi->xdp_rings[q]);
 840                         if (!p)
 841                                 continue;
 842
 843                         do {
 844                                 start = u64_stats_fetch_begin_irq(&p->syncp);
 845                                 packets = p->stats.packets;
 846                                 bytes = p->stats.bytes;
 847                         } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 848                         tx_b += bytes;
 849                         tx_p += packets;
 850                         tx_restart += p->tx_stats.restart_queue;
 851                         tx_busy += p->tx_stats.tx_busy;
 852                         tx_linearize += p->tx_stats.tx_linearize;
 853                         tx_force_wb += p->tx_stats.tx_force_wb;
 854                 }
 855         }
 856         rcu_read_unlock();
 857         vsi->tx_restart = tx_restart;
 858         vsi->tx_busy = tx_busy;
 859         vsi->tx_linearize = tx_linearize;
 860         vsi->tx_force_wb = tx_force_wb;
 861         vsi->rx_page_failed = rx_page;
 862         vsi->rx_buf_failed = rx_buf;
 863
 864         ns->rx_packets = rx_p;
 865         ns->rx_bytes = rx_b;
 866         ns->tx_packets = tx_p;
 867         ns->tx_bytes = tx_b;
 868
 869         /* update netdev stats from eth stats */
 870         i40e_update_eth_stats(vsi);
 871         ons->tx_errors = oes->tx_errors;
 872         ns->tx_errors = es->tx_errors;
 873         ons->multicast = oes->rx_multicast;
 874         ns->multicast = es->rx_multicast;
 875         ons->rx_dropped = oes->rx_discards;
 876         ns->rx_dropped = es->rx_discards;
 877         ons->tx_dropped = oes->tx_discards;
 878         ns->tx_dropped = es->tx_discards;
 879
 880         /* pull in a couple PF stats if this is the main vsi */
 881         if (vsi == pf->vsi[pf->lan_vsi]) {
 882                 ns->rx_crc_errors = pf->stats.crc_errors;
 883                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 884                 ns->rx_length_errors = pf->stats.rx_length_errors;
 885         }
 886 }
 887
 888 /**
 889  * i40e_update_pf_stats - Update the PF statistics counters.
 890  * @pf: the PF to be updated
 891  **/
 892 static void i40e_update_pf_stats(struct i40e_pf *pf)
 893 {
 894         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 895         struct i40e_hw_port_stats *nsd = &pf->stats;
 896         struct i40e_hw *hw = &pf->hw;
 897         u32 val;
 898         int i;
 899
 900         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 901                            I40E_GLPRT_GORCL(hw->port),
 902                            pf->stat_offsets_loaded,
 903                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 904         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 905                            I40E_GLPRT_GOTCL(hw->port),
 906                            pf->stat_offsets_loaded,
 907                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 908         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 909                            pf->stat_offsets_loaded,
 910                            &osd->eth.rx_discards,
 911                            &nsd->eth.rx_discards);
 912         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 913                            I40E_GLPRT_UPRCL(hw->port),
 914                            pf->stat_offsets_loaded,
 915                            &osd->eth.rx_unicast,
 916                            &nsd->eth.rx_unicast);
 917         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 918                            I40E_GLPRT_MPRCL(hw->port),
 919                            pf->stat_offsets_loaded,
 920                            &osd->eth.rx_multicast,
 921                            &nsd->eth.rx_multicast);
 922         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 923                            I40E_GLPRT_BPRCL(hw->port),
 924                            pf->stat_offsets_loaded,
 925                            &osd->eth.rx_broadcast,
 926                            &nsd->eth.rx_broadcast);
 927         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 928                            I40E_GLPRT_UPTCL(hw->port),
 929                            pf->stat_offsets_loaded,
 930                            &osd->eth.tx_unicast,
 931                            &nsd->eth.tx_unicast);
 932         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 933                            I40E_GLPRT_MPTCL(hw->port),
 934                            pf->stat_offsets_loaded,
 935                            &osd->eth.tx_multicast,
 936                            &nsd->eth.tx_multicast);
 937         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 938                            I40E_GLPRT_BPTCL(hw->port),
 939                            pf->stat_offsets_loaded,
 940                            &osd->eth.tx_broadcast,
 941                            &nsd->eth.tx_broadcast);
 942
 943         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->tx_dropped_link_down,
 946                            &nsd->tx_dropped_link_down);
 947
 948         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 949                            pf->stat_offsets_loaded,
 950                            &osd->crc_errors, &nsd->crc_errors);
 951
 952         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 953                            pf->stat_offsets_loaded,
 954                            &osd->illegal_bytes, &nsd->illegal_bytes);
 955
 956         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 957                            pf->stat_offsets_loaded,
 958                            &osd->mac_local_faults,
 959                            &nsd->mac_local_faults);
 960         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 961                            pf->stat_offsets_loaded,
 962                            &osd->mac_remote_faults,
 963                            &nsd->mac_remote_faults);
 964
 965         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 966                            pf->stat_offsets_loaded,
 967                            &osd->rx_length_errors,
 968                            &nsd->rx_length_errors);
 969
 970         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 971                            pf->stat_offsets_loaded,
 972                            &osd->link_xon_rx, &nsd->link_xon_rx);
 973         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 974                            pf->stat_offsets_loaded,
 975                            &osd->link_xon_tx, &nsd->link_xon_tx);
 976         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 977                            pf->stat_offsets_loaded,
 978                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 979         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 980                            pf->stat_offsets_loaded,
 981                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 982
 983         for (i = 0; i < 8; i++) {
 984                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 985                                    pf->stat_offsets_loaded,
 986                                    &osd->priority_xoff_rx[i],
 987                                    &nsd->priority_xoff_rx[i]);
 988                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 989                                    pf->stat_offsets_loaded,
 990                                    &osd->priority_xon_rx[i],
 991                                    &nsd->priority_xon_rx[i]);
 992                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 993                                    pf->stat_offsets_loaded,
 994                                    &osd->priority_xon_tx[i],
 995                                    &nsd->priority_xon_tx[i]);
 996                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 997                                    pf->stat_offsets_loaded,
 998                                    &osd->priority_xoff_tx[i],
 999                                    &nsd->priority_xoff_tx[i]);
1000                 i40e_stat_update32(hw,
1001                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1002                                    pf->stat_offsets_loaded,
1003                                    &osd->priority_xon_2_xoff[i],
1004                                    &nsd->priority_xon_2_xoff[i]);
1005         }
1006
1007         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1008                            I40E_GLPRT_PRC64L(hw->port),
1009                            pf->stat_offsets_loaded,
1010                            &osd->rx_size_64, &nsd->rx_size_64);
1011         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1012                            I40E_GLPRT_PRC127L(hw->port),
1013                            pf->stat_offsets_loaded,
1014                            &osd->rx_size_127, &nsd->rx_size_127);
1015         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1016                            I40E_GLPRT_PRC255L(hw->port),
1017                            pf->stat_offsets_loaded,
1018                            &osd->rx_size_255, &nsd->rx_size_255);
1019         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1020                            I40E_GLPRT_PRC511L(hw->port),
1021                            pf->stat_offsets_loaded,
1022                            &osd->rx_size_511, &nsd->rx_size_511);
1023         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1024                            I40E_GLPRT_PRC1023L(hw->port),
1025                            pf->stat_offsets_loaded,
1026                            &osd->rx_size_1023, &nsd->rx_size_1023);
1027         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1028                            I40E_GLPRT_PRC1522L(hw->port),
1029                            pf->stat_offsets_loaded,
1030                            &osd->rx_size_1522, &nsd->rx_size_1522);
1031         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1032                            I40E_GLPRT_PRC9522L(hw->port),
1033                            pf->stat_offsets_loaded,
1034                            &osd->rx_size_big, &nsd->rx_size_big);
1035
1036         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1037                            I40E_GLPRT_PTC64L(hw->port),
1038                            pf->stat_offsets_loaded,
1039                            &osd->tx_size_64, &nsd->tx_size_64);
1040         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1041                            I40E_GLPRT_PTC127L(hw->port),
1042                            pf->stat_offsets_loaded,
1043                            &osd->tx_size_127, &nsd->tx_size_127);
1044         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1045                            I40E_GLPRT_PTC255L(hw->port),
1046                            pf->stat_offsets_loaded,
1047                            &osd->tx_size_255, &nsd->tx_size_255);
1048         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1049                            I40E_GLPRT_PTC511L(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->tx_size_511, &nsd->tx_size_511);
1052         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1053                            I40E_GLPRT_PTC1023L(hw->port),
1054                            pf->stat_offsets_loaded,
1055                            &osd->tx_size_1023, &nsd->tx_size_1023);
1056         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1057                            I40E_GLPRT_PTC1522L(hw->port),
1058                            pf->stat_offsets_loaded,
1059                            &osd->tx_size_1522, &nsd->tx_size_1522);
1060         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1061                            I40E_GLPRT_PTC9522L(hw->port),
1062                            pf->stat_offsets_loaded,
1063                            &osd->tx_size_big, &nsd->tx_size_big);
1064
1065         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1066                            pf->stat_offsets_loaded,
1067                            &osd->rx_undersize, &nsd->rx_undersize);
1068         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1069                            pf->stat_offsets_loaded,
1070                            &osd->rx_fragments, &nsd->rx_fragments);
1071         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1072                            pf->stat_offsets_loaded,
1073                            &osd->rx_oversize, &nsd->rx_oversize);
1074         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1075                            pf->stat_offsets_loaded,
1076                            &osd->rx_jabber, &nsd->rx_jabber);
1077
1078         /* FDIR stats */
1079         i40e_stat_update_and_clear32(hw,
1080                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1081                         &nsd->fd_atr_match);
1082         i40e_stat_update_and_clear32(hw,
1083                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1084                         &nsd->fd_sb_match);
1085         i40e_stat_update_and_clear32(hw,
1086                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1087                         &nsd->fd_atr_tunnel_match);
1088
1089         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1090         nsd->tx_lpi_status =
1091                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1092                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1093         nsd->rx_lpi_status =
1094                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1095                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1096         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1097                            pf->stat_offsets_loaded,
1098                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1099         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1100                            pf->stat_offsets_loaded,
1101                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1102
1103         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1104             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1105                 nsd->fd_sb_status = true;
1106         else
1107                 nsd->fd_sb_status = false;
1108
1109         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1110             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1111                 nsd->fd_atr_status = true;
1112         else
1113                 nsd->fd_atr_status = false;
1114
1115         pf->stat_offsets_loaded = true;
1116 }
1117
1118 /**
1119  * i40e_update_stats - Update the various statistics counters.
1120  * @vsi: the VSI to be updated
1121  *
1122  * Update the various stats for this VSI and its related entities.
1123  **/
1124 void i40e_update_stats(struct i40e_vsi *vsi)
1125 {
1126         struct i40e_pf *pf = vsi->back;
1127
1128         if (vsi == pf->vsi[pf->lan_vsi])
1129                 i40e_update_pf_stats(pf);
1130
1131         i40e_update_vsi_stats(vsi);
1132 }
1133
1134 /**
1135  * i40e_count_filters - counts VSI mac filters
1136  * @vsi: the VSI to be searched
1137  *
1138  * Returns count of mac filters
1139  **/
1140 int i40e_count_filters(struct i40e_vsi *vsi)
1141 {
1142         struct i40e_mac_filter *f;
1143         struct hlist_node *h;
1144         int bkt;
1145         int cnt = 0;
1146
1147         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1148                 ++cnt;
1149
1150         return cnt;
1151 }
1152
1153 /**
1154  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1155  * @vsi: the VSI to be searched
1156  * @macaddr: the MAC address
1157  * @vlan: the vlan
1158  *
1159  * Returns ptr to the filter object or NULL
1160  **/
1161 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1162                                                 const u8 *macaddr, s16 vlan)
1163 {
1164         struct i40e_mac_filter *f;
1165         u64 key;
1166
1167         if (!vsi || !macaddr)
1168                 return NULL;
1169
1170         key = i40e_addr_to_hkey(macaddr);
1171         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1172                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1173                     (vlan == f->vlan))
1174                         return f;
1175         }
1176         return NULL;
1177 }
1178
1179 /**
1180  * i40e_find_mac - Find a mac addr in the macvlan filters list
1181  * @vsi: the VSI to be searched
1182  * @macaddr: the MAC address we are searching for
1183  *
1184  * Returns the first filter with the provided MAC address or NULL if
1185  * MAC address was not found
1186  **/
1187 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1188 {
1189         struct i40e_mac_filter *f;
1190         u64 key;
1191
1192         if (!vsi || !macaddr)
1193                 return NULL;
1194
1195         key = i40e_addr_to_hkey(macaddr);
1196         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1197                 if ((ether_addr_equal(macaddr, f->macaddr)))
1198                         return f;
1199         }
1200         return NULL;
1201 }
1202
1203 /**
1204  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1205  * @vsi: the VSI to be searched
1206  *
1207  * Returns true if VSI is in vlan mode or false otherwise
1208  **/
1209 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1210 {
1211         /* If we have a PVID, always operate in VLAN mode */
1212         if (vsi->info.pvid)
1213                 return true;
1214
1215         /* We need to operate in VLAN mode whenever we have any filters with
1216          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1217          * time, incurring search cost repeatedly. However, we can notice two
1218          * things:
1219          *
1220          * 1) the only place where we can gain a VLAN filter is in
1221          *    i40e_add_filter.
1222          *
1223          * 2) the only place where filters are actually removed is in
1224          *    i40e_sync_filters_subtask.
1225          *
1226          * Thus, we can simply use a boolean value, has_vlan_filters which we
1227          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1228          * we have to perform the full search after deleting filters in
1229          * i40e_sync_filters_subtask, but we already have to search
1230          * filters here and can perform the check at the same time. This
1231          * results in avoiding embedding a loop for VLAN mode inside another
1232          * loop over all the filters, and should maintain correctness as noted
1233          * above.
1234          */
1235         return vsi->has_vlan_filter;
1236 }
1237
1238 /**
1239  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1240  * @vsi: the VSI to configure
1241  * @tmp_add_list: list of filters ready to be added
1242  * @tmp_del_list: list of filters ready to be deleted
1243  * @vlan_filters: the number of active VLAN filters
1244  *
1245  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1246  * behave as expected. If we have any active VLAN filters remaining or about
1247  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1248  * so that they only match against untagged traffic. If we no longer have any
1249  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1250  * so that they match against both tagged and untagged traffic. In this way,
1251  * we ensure that we correctly receive the desired traffic. This ensures that
1252  * when we have an active VLAN we will receive only untagged traffic and
1253  * traffic matching active VLANs. If we have no active VLANs then we will
1254  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1255  *
1256  * Finally, in a similar fashion, this function also corrects filters when
1257  * there is an active PVID assigned to this VSI.
1258  *
1259  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1260  *
1261  * This function is only expected to be called from within
1262  * i40e_sync_vsi_filters.
1263  *
1264  * NOTE: This function expects to be called while under the
1265  * mac_filter_hash_lock
1266  */
1267 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1268                                          struct hlist_head *tmp_add_list,
1269                                          struct hlist_head *tmp_del_list,
1270                                          int vlan_filters)
1271 {
1272         s16 pvid = le16_to_cpu(vsi->info.pvid);
1273         struct i40e_mac_filter *f, *add_head;
1274         struct i40e_new_mac_filter *new;
1275         struct hlist_node *h;
1276         int bkt, new_vlan;
1277
1278         /* To determine if a particular filter needs to be replaced we
1279          * have the three following conditions:
1280          *
1281          * a) if we have a PVID assigned, then all filters which are
1282          *    not marked as VLAN=PVID must be replaced with filters that
1283          *    are.
1284          * b) otherwise, if we have any active VLANS, all filters
1285          *    which are marked as VLAN=-1 must be replaced with
1286          *    filters marked as VLAN=0
1287          * c) finally, if we do not have any active VLANS, all filters
1288          *    which are marked as VLAN=0 must be replaced with filters
1289          *    marked as VLAN=-1
1290          */
1291
1292         /* Update the filters about to be added in place */
1293         hlist_for_each_entry(new, tmp_add_list, hlist) {
1294                 if (pvid && new->f->vlan != pvid)
1295                         new->f->vlan = pvid;
1296                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1297                         new->f->vlan = 0;
1298                 else if (!vlan_filters && new->f->vlan == 0)
1299                         new->f->vlan = I40E_VLAN_ANY;
1300         }
1301
1302         /* Update the remaining active filters */
1303         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1304                 /* Combine the checks for whether a filter needs to be changed
1305                  * and then determine the new VLAN inside the if block, in
1306                  * order to avoid duplicating code for adding the new filter
1307                  * then deleting the old filter.
1308                  */
1309                 if ((pvid && f->vlan != pvid) ||
1310                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1311                     (!vlan_filters && f->vlan == 0)) {
1312                         /* Determine the new vlan we will be adding */
1313                         if (pvid)
1314                                 new_vlan = pvid;
1315                         else if (vlan_filters)
1316                                 new_vlan = 0;
1317                         else
1318                                 new_vlan = I40E_VLAN_ANY;
1319
1320                         /* Create the new filter */
1321                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1322                         if (!add_head)
1323                                 return -ENOMEM;
1324
1325                         /* Create a temporary i40e_new_mac_filter */
1326                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1327                         if (!new)
1328                                 return -ENOMEM;
1329
1330                         new->f = add_head;
1331                         new->state = add_head->state;
1332
1333                         /* Add the new filter to the tmp list */
1334                         hlist_add_head(&new->hlist, tmp_add_list);
1335
1336                         /* Put the original filter into the delete list */
1337                         f->state = I40E_FILTER_REMOVE;
1338                         hash_del(&f->hlist);
1339                         hlist_add_head(&f->hlist, tmp_del_list);
1340                 }
1341         }
1342
1343         vsi->has_vlan_filter = !!vlan_filters;
1344
1345         return 0;
1346 }
1347
1348 /**
1349  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1350  * @vsi: the PF Main VSI - inappropriate for any other VSI
1351  * @macaddr: the MAC address
1352  *
1353  * Remove whatever filter the firmware set up so the driver can manage
1354  * its own filtering intelligently.
1355  **/
1356 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1357 {
1358         struct i40e_aqc_remove_macvlan_element_data element;
1359         struct i40e_pf *pf = vsi->back;
1360
1361         /* Only appropriate for the PF main VSI */
1362         if (vsi->type != I40E_VSI_MAIN)
1363                 return;
1364
1365         memset(&element, 0, sizeof(element));
1366         ether_addr_copy(element.mac_addr, macaddr);
1367         element.vlan_tag = 0;
1368         /* Ignore error returns, some firmware does it this way... */
1369         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1370         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1371
1372         memset(&element, 0, sizeof(element));
1373         ether_addr_copy(element.mac_addr, macaddr);
1374         element.vlan_tag = 0;
1375         /* ...and some firmware does it this way. */
1376         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1377                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1378         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1379 }
1380
1381 /**
1382  * i40e_add_filter - Add a mac/vlan filter to the VSI
1383  * @vsi: the VSI to be searched
1384  * @macaddr: the MAC address
1385  * @vlan: the vlan
1386  *
1387  * Returns ptr to the filter object or NULL when no memory available.
1388  *
1389  * NOTE: This function is expected to be called with mac_filter_hash_lock
1390  * being held.
1391  **/
1392 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1393                                         const u8 *macaddr, s16 vlan)
1394 {
1395         struct i40e_mac_filter *f;
1396         u64 key;
1397
1398         if (!vsi || !macaddr)
1399                 return NULL;
1400
1401         f = i40e_find_filter(vsi, macaddr, vlan);
1402         if (!f) {
1403                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1404                 if (!f)
1405                         return NULL;
1406
1407                 /* Update the boolean indicating if we need to function in
1408                  * VLAN mode.
1409                  */
1410                 if (vlan >= 0)
1411                         vsi->has_vlan_filter = true;
1412
1413                 ether_addr_copy(f->macaddr, macaddr);
1414                 f->vlan = vlan;
1415                 f->state = I40E_FILTER_NEW;
1416                 INIT_HLIST_NODE(&f->hlist);
1417
1418                 key = i40e_addr_to_hkey(macaddr);
1419                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1420
1421                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1422                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1423         }
1424
1425         /* If we're asked to add a filter that has been marked for removal, it
1426          * is safe to simply restore it to active state. __i40e_del_filter
1427          * will have simply deleted any filters which were previously marked
1428          * NEW or FAILED, so if it is currently marked REMOVE it must have
1429          * previously been ACTIVE. Since we haven't yet run the sync filters
1430          * task, just restore this filter to the ACTIVE state so that the
1431          * sync task leaves it in place
1432          */
1433         if (f->state == I40E_FILTER_REMOVE)
1434                 f->state = I40E_FILTER_ACTIVE;
1435
1436         return f;
1437 }
1438
1439 /**
1440  * __i40e_del_filter - Remove a specific filter from the VSI
1441  * @vsi: VSI to remove from
1442  * @f: the filter to remove from the list
1443  *
1444  * This function should be called instead of i40e_del_filter only if you know
1445  * the exact filter you will remove already, such as via i40e_find_filter or
1446  * i40e_find_mac.
1447  *
1448  * NOTE: This function is expected to be called with mac_filter_hash_lock
1449  * being held.
1450  * ANOTHER NOTE: This function MUST be called from within the context of
1451  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1452  * instead of list_for_each_entry().
1453  **/
1454 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1455 {
1456         if (!f)
1457                 return;
1458
1459         /* If the filter was never added to firmware then we can just delete it
1460          * directly and we don't want to set the status to remove or else an
1461          * admin queue command will unnecessarily fire.
1462          */
1463         if ((f->state == I40E_FILTER_FAILED) ||
1464             (f->state == I40E_FILTER_NEW)) {
1465                 hash_del(&f->hlist);
1466                 kfree(f);
1467         } else {
1468                 f->state = I40E_FILTER_REMOVE;
1469         }
1470
1471         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1472         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1473 }
1474
1475 /**
1476  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1477  * @vsi: the VSI to be searched
1478  * @macaddr: the MAC address
1479  * @vlan: the VLAN
1480  *
1481  * NOTE: This function is expected to be called with mac_filter_hash_lock
1482  * being held.
1483  * ANOTHER NOTE: This function MUST be called from within the context of
1484  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1485  * instead of list_for_each_entry().
1486  **/
1487 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1488 {
1489         struct i40e_mac_filter *f;
1490
1491         if (!vsi || !macaddr)
1492                 return;
1493
1494         f = i40e_find_filter(vsi, macaddr, vlan);
1495         __i40e_del_filter(vsi, f);
1496 }
1497
1498 /**
1499  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1500  * @vsi: the VSI to be searched
1501  * @macaddr: the mac address to be filtered
1502  *
1503  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1504  * go through all the macvlan filters and add a macvlan filter for each
1505  * unique vlan that already exists. If a PVID has been assigned, instead only
1506  * add the macaddr to that VLAN.
1507  *
1508  * Returns last filter added on success, else NULL
1509  **/
1510 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1511                                             const u8 *macaddr)
1512 {
1513         struct i40e_mac_filter *f, *add = NULL;
1514         struct hlist_node *h;
1515         int bkt;
1516
1517         if (vsi->info.pvid)
1518                 return i40e_add_filter(vsi, macaddr,
1519                                        le16_to_cpu(vsi->info.pvid));
1520
1521         if (!i40e_is_vsi_in_vlan(vsi))
1522                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1523
1524         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1525                 if (f->state == I40E_FILTER_REMOVE)
1526                         continue;
1527                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1528                 if (!add)
1529                         return NULL;
1530         }
1531
1532         return add;
1533 }
1534
1535 /**
1536  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1537  * @vsi: the VSI to be searched
1538  * @macaddr: the mac address to be removed
1539  *
1540  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1541  * associated with.
1542  *
1543  * Returns 0 for success, or error
1544  **/
1545 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1546 {
1547         struct i40e_mac_filter *f;
1548         struct hlist_node *h;
1549         bool found = false;
1550         int bkt;
1551
1552         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1553         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1554                 if (ether_addr_equal(macaddr, f->macaddr)) {
1555                         __i40e_del_filter(vsi, f);
1556                         found = true;
1557                 }
1558         }
1559
1560         if (found)
1561                 return 0;
1562         else
1563                 return -ENOENT;
1564 }
1565
1566 /**
1567  * i40e_set_mac - NDO callback to set mac address
1568  * @netdev: network interface device structure
1569  * @p: pointer to an address structure
1570  *
1571  * Returns 0 on success, negative on failure
1572  **/
1573 static int i40e_set_mac(struct net_device *netdev, void *p)
1574 {
1575         struct i40e_netdev_priv *np = netdev_priv(netdev);
1576         struct i40e_vsi *vsi = np->vsi;
1577         struct i40e_pf *pf = vsi->back;
1578         struct i40e_hw *hw = &pf->hw;
1579         struct sockaddr *addr = p;
1580
1581         if (!is_valid_ether_addr(addr->sa_data))
1582                 return -EADDRNOTAVAIL;
1583
1584         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1585                 netdev_info(netdev, "already using mac address %pM\n",
1586                             addr->sa_data);
1587                 return 0;
1588         }
1589
1590         if (test_bit(__I40E_DOWN, pf->state) ||
1591             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1592                 return -EADDRNOTAVAIL;
1593
1594         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1595                 netdev_info(netdev, "returning to hw mac address %pM\n",
1596                             hw->mac.addr);
1597         else
1598                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1599
1600         /* Copy the address first, so that we avoid a possible race with
1601          * .set_rx_mode().
1602          * - Remove old address from MAC filter
1603          * - Copy new address
1604          * - Add new address to MAC filter
1605          */
1606         spin_lock_bh(&vsi->mac_filter_hash_lock);
1607         i40e_del_mac_filter(vsi, netdev->dev_addr);
1608         eth_hw_addr_set(netdev, addr->sa_data);
1609         i40e_add_mac_filter(vsi, netdev->dev_addr);
1610         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1611
1612         if (vsi->type == I40E_VSI_MAIN) {
1613                 i40e_status ret;
1614
1615                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1616                                                 addr->sa_data, NULL);
1617                 if (ret)
1618                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1619                                     i40e_stat_str(hw, ret),
1620                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1621         }
1622
1623         /* schedule our worker thread which will take care of
1624          * applying the new filter changes
1625          */
1626         i40e_service_event_schedule(pf);
1627         return 0;
1628 }
1629
1630 /**
1631  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1632  * @vsi: vsi structure
1633  * @seed: RSS hash seed
1634  * @lut: pointer to lookup table of lut_size
1635  * @lut_size: size of the lookup table
1636  **/
1637 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1638                               u8 *lut, u16 lut_size)
1639 {
1640         struct i40e_pf *pf = vsi->back;
1641         struct i40e_hw *hw = &pf->hw;
1642         int ret = 0;
1643
1644         if (seed) {
1645                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1646                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1647                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1648                 if (ret) {
1649                         dev_info(&pf->pdev->dev,
1650                                  "Cannot set RSS key, err %s aq_err %s\n",
1651                                  i40e_stat_str(hw, ret),
1652                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1653                         return ret;
1654                 }
1655         }
1656         if (lut) {
1657                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1658
1659                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1660                 if (ret) {
1661                         dev_info(&pf->pdev->dev,
1662                                  "Cannot set RSS lut, err %s aq_err %s\n",
1663                                  i40e_stat_str(hw, ret),
1664                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1665                         return ret;
1666                 }
1667         }
1668         return ret;
1669 }
1670
1671 /**
1672  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1673  * @vsi: VSI structure
1674  **/
1675 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1676 {
1677         struct i40e_pf *pf = vsi->back;
1678         u8 seed[I40E_HKEY_ARRAY_SIZE];
1679         u8 *lut;
1680         int ret;
1681
1682         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1683                 return 0;
1684         if (!vsi->rss_size)
1685                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1686                                       vsi->num_queue_pairs);
1687         if (!vsi->rss_size)
1688                 return -EINVAL;
1689         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1690         if (!lut)
1691                 return -ENOMEM;
1692
1693         /* Use the user configured hash keys and lookup table if there is one,
1694          * otherwise use default
1695          */
1696         if (vsi->rss_lut_user)
1697                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1698         else
1699                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1700         if (vsi->rss_hkey_user)
1701                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1702         else
1703                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1704         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1705         kfree(lut);
1706         return ret;
1707 }
1708
1709 /**
1710  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1711  * @vsi: the VSI being configured,
1712  * @ctxt: VSI context structure
1713  * @enabled_tc: number of traffic classes to enable
1714  *
1715  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1716  **/
1717 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1718                                            struct i40e_vsi_context *ctxt,
1719                                            u8 enabled_tc)
1720 {
1721         u16 qcount = 0, max_qcount, qmap, sections = 0;
1722         int i, override_q, pow, num_qps, ret;
1723         u8 netdev_tc = 0, offset = 0;
1724
1725         if (vsi->type != I40E_VSI_MAIN)
1726                 return -EINVAL;
1727         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1728         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1729         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1730         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1731         num_qps = vsi->mqprio_qopt.qopt.count[0];
1732
1733         /* find the next higher power-of-2 of num queue pairs */
1734         pow = ilog2(num_qps);
1735         if (!is_power_of_2(num_qps))
1736                 pow++;
1737         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1738                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1739
1740         /* Setup queue offset/count for all TCs for given VSI */
1741         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1742         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1743                 /* See if the given TC is enabled for the given VSI */
1744                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1745                         offset = vsi->mqprio_qopt.qopt.offset[i];
1746                         qcount = vsi->mqprio_qopt.qopt.count[i];
1747                         if (qcount > max_qcount)
1748                                 max_qcount = qcount;
1749                         vsi->tc_config.tc_info[i].qoffset = offset;
1750                         vsi->tc_config.tc_info[i].qcount = qcount;
1751                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1752                 } else {
1753                         /* TC is not enabled so set the offset to
1754                          * default queue and allocate one queue
1755                          * for the given TC.
1756                          */
1757                         vsi->tc_config.tc_info[i].qoffset = 0;
1758                         vsi->tc_config.tc_info[i].qcount = 1;
1759                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1760                 }
1761         }
1762
1763         /* Set actual Tx/Rx queue pairs */
1764         vsi->num_queue_pairs = offset + qcount;
1765
1766         /* Setup queue TC[0].qmap for given VSI context */
1767         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1768         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1769         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1770         ctxt->info.valid_sections |= cpu_to_le16(sections);
1771
1772         /* Reconfigure RSS for main VSI with max queue count */
1773         vsi->rss_size = max_qcount;
1774         ret = i40e_vsi_config_rss(vsi);
1775         if (ret) {
1776                 dev_info(&vsi->back->pdev->dev,
1777                          "Failed to reconfig rss for num_queues (%u)\n",
1778                          max_qcount);
1779                 return ret;
1780         }
1781         vsi->reconfig_rss = true;
1782         dev_dbg(&vsi->back->pdev->dev,
1783                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1784
1785         /* Find queue count available for channel VSIs and starting offset
1786          * for channel VSIs
1787          */
1788         override_q = vsi->mqprio_qopt.qopt.count[0];
1789         if (override_q && override_q < vsi->num_queue_pairs) {
1790                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1791                 vsi->next_base_queue = override_q;
1792         }
1793         return 0;
1794 }
1795
1796 /**
1797  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1798  * @vsi: the VSI being setup
1799  * @ctxt: VSI context structure
1800  * @enabled_tc: Enabled TCs bitmap
1801  * @is_add: True if called before Add VSI
1802  *
1803  * Setup VSI queue mapping for enabled traffic classes.
1804  **/
1805 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1806                                      struct i40e_vsi_context *ctxt,
1807                                      u8 enabled_tc,
1808                                      bool is_add)
1809 {
1810         struct i40e_pf *pf = vsi->back;
1811         u16 num_tc_qps = 0;
1812         u16 sections = 0;
1813         u8 netdev_tc = 0;
1814         u16 numtc = 1;
1815         u16 qcount;
1816         u8 offset;
1817         u16 qmap;
1818         int i;
1819
1820         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1821         offset = 0;
1822         /* zero out queue mapping, it will get updated on the end of the function */
1823         memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
1824
1825         if (vsi->type == I40E_VSI_MAIN) {
1826                 /* This code helps add more queue to the VSI if we have
1827                  * more cores than RSS can support, the higher cores will
1828                  * be served by ATR or other filters. Furthermore, the
1829                  * non-zero req_queue_pairs says that user requested a new
1830                  * queue count via ethtool's set_channels, so use this
1831                  * value for queues distribution across traffic classes
1832                  */
1833                 if (vsi->req_queue_pairs > 0)
1834                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1835                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1836                         vsi->num_queue_pairs = pf->num_lan_msix;
1837         }
1838
1839         /* Number of queues per enabled TC */
1840         if (vsi->type == I40E_VSI_MAIN ||
1841             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
1842                 num_tc_qps = vsi->num_queue_pairs;
1843         else
1844                 num_tc_qps = vsi->alloc_queue_pairs;
1845
1846         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1847                 /* Find numtc from enabled TC bitmap */
1848                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1849                         if (enabled_tc & BIT(i)) /* TC is enabled */
1850                                 numtc++;
1851                 }
1852                 if (!numtc) {
1853                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1854                         numtc = 1;
1855                 }
1856                 num_tc_qps = num_tc_qps / numtc;
1857                 num_tc_qps = min_t(int, num_tc_qps,
1858                                    i40e_pf_get_max_q_per_tc(pf));
1859         }
1860
1861         vsi->tc_config.numtc = numtc;
1862         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1863
1864         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1865         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1866                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1867
1868         /* Setup queue offset/count for all TCs for given VSI */
1869         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1870                 /* See if the given TC is enabled for the given VSI */
1871                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1872                         /* TC is enabled */
1873                         int pow, num_qps;
1874
1875                         switch (vsi->type) {
1876                         case I40E_VSI_MAIN:
1877                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1878                                     I40E_FLAG_FD_ATR_ENABLED)) ||
1879                                     vsi->tc_config.enabled_tc != 1) {
1880                                         qcount = min_t(int, pf->alloc_rss_size,
1881                                                        num_tc_qps);
1882                                         break;
1883                                 }
1884                                 fallthrough;
1885                         case I40E_VSI_FDIR:
1886                         case I40E_VSI_SRIOV:
1887                         case I40E_VSI_VMDQ2:
1888                         default:
1889                                 qcount = num_tc_qps;
1890                                 WARN_ON(i != 0);
1891                                 break;
1892                         }
1893                         vsi->tc_config.tc_info[i].qoffset = offset;
1894                         vsi->tc_config.tc_info[i].qcount = qcount;
1895
1896                         /* find the next higher power-of-2 of num queue pairs */
1897                         num_qps = qcount;
1898                         pow = 0;
1899                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1900                                 pow++;
1901                                 num_qps >>= 1;
1902                         }
1903
1904                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1905                         qmap =
1906                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1907                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1908
1909                         offset += qcount;
1910                 } else {
1911                         /* TC is not enabled so set the offset to
1912                          * default queue and allocate one queue
1913                          * for the given TC.
1914                          */
1915                         vsi->tc_config.tc_info[i].qoffset = 0;
1916                         vsi->tc_config.tc_info[i].qcount = 1;
1917                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1918
1919                         qmap = 0;
1920                 }
1921                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1922         }
1923         /* Do not change previously set num_queue_pairs for PFs and VFs*/
1924         if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
1925             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
1926             (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
1927                 vsi->num_queue_pairs = offset;
1928
1929         /* Scheduler section valid can only be set for ADD VSI */
1930         if (is_add) {
1931                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1932
1933                 ctxt->info.up_enable_bits = enabled_tc;
1934         }
1935         if (vsi->type == I40E_VSI_SRIOV) {
1936                 ctxt->info.mapping_flags |=
1937                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1938                 for (i = 0; i < vsi->num_queue_pairs; i++)
1939                         ctxt->info.queue_mapping[i] =
1940                                                cpu_to_le16(vsi->base_queue + i);
1941         } else {
1942                 ctxt->info.mapping_flags |=
1943                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1944                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1945         }
1946         ctxt->info.valid_sections |= cpu_to_le16(sections);
1947 }
1948
1949 /**
1950  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1951  * @netdev: the netdevice
1952  * @addr: address to add
1953  *
1954  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1955  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1956  */
1957 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1958 {
1959         struct i40e_netdev_priv *np = netdev_priv(netdev);
1960         struct i40e_vsi *vsi = np->vsi;
1961
1962         if (i40e_add_mac_filter(vsi, addr))
1963                 return 0;
1964         else
1965                 return -ENOMEM;
1966 }
1967
1968 /**
1969  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1970  * @netdev: the netdevice
1971  * @addr: address to add
1972  *
1973  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1974  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1975  */
1976 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1977 {
1978         struct i40e_netdev_priv *np = netdev_priv(netdev);
1979         struct i40e_vsi *vsi = np->vsi;
1980
1981         /* Under some circumstances, we might receive a request to delete
1982          * our own device address from our uc list. Because we store the
1983          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1984          * such requests and not delete our device address from this list.
1985          */
1986         if (ether_addr_equal(addr, netdev->dev_addr))
1987                 return 0;
1988
1989         i40e_del_mac_filter(vsi, addr);
1990
1991         return 0;
1992 }
1993
1994 /**
1995  * i40e_set_rx_mode - NDO callback to set the netdev filters
1996  * @netdev: network interface device structure
1997  **/
1998 static void i40e_set_rx_mode(struct net_device *netdev)
1999 {
2000         struct i40e_netdev_priv *np = netdev_priv(netdev);
2001         struct i40e_vsi *vsi = np->vsi;
2002
2003         spin_lock_bh(&vsi->mac_filter_hash_lock);
2004
2005         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2006         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2007
2008         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2009
2010         /* check for other flag changes */
2011         if (vsi->current_netdev_flags != vsi->netdev->flags) {
2012                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2013                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2014         }
2015 }
2016
2017 /**
2018  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2019  * @vsi: Pointer to VSI struct
2020  * @from: Pointer to list which contains MAC filter entries - changes to
2021  *        those entries needs to be undone.
2022  *
2023  * MAC filter entries from this list were slated for deletion.
2024  **/
2025 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2026                                          struct hlist_head *from)
2027 {
2028         struct i40e_mac_filter *f;
2029         struct hlist_node *h;
2030
2031         hlist_for_each_entry_safe(f, h, from, hlist) {
2032                 u64 key = i40e_addr_to_hkey(f->macaddr);
2033
2034                 /* Move the element back into MAC filter list*/
2035                 hlist_del(&f->hlist);
2036                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2037         }
2038 }
2039
2040 /**
2041  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2042  * @vsi: Pointer to vsi struct
2043  * @from: Pointer to list which contains MAC filter entries - changes to
2044  *        those entries needs to be undone.
2045  *
2046  * MAC filter entries from this list were slated for addition.
2047  **/
2048 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2049                                          struct hlist_head *from)
2050 {
2051         struct i40e_new_mac_filter *new;
2052         struct hlist_node *h;
2053
2054         hlist_for_each_entry_safe(new, h, from, hlist) {
2055                 /* We can simply free the wrapper structure */
2056                 hlist_del(&new->hlist);
2057                 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2058                 kfree(new);
2059         }
2060 }
2061
2062 /**
2063  * i40e_next_filter - Get the next non-broadcast filter from a list
2064  * @next: pointer to filter in list
2065  *
2066  * Returns the next non-broadcast filter in the list. Required so that we
2067  * ignore broadcast filters within the list, since these are not handled via
2068  * the normal firmware update path.
2069  */
2070 static
2071 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2072 {
2073         hlist_for_each_entry_continue(next, hlist) {
2074                 if (!is_broadcast_ether_addr(next->f->macaddr))
2075                         return next;
2076         }
2077
2078         return NULL;
2079 }
2080
2081 /**
2082  * i40e_update_filter_state - Update filter state based on return data
2083  * from firmware
2084  * @count: Number of filters added
2085  * @add_list: return data from fw
2086  * @add_head: pointer to first filter in current batch
2087  *
2088  * MAC filter entries from list were slated to be added to device. Returns
2089  * number of successful filters. Note that 0 does NOT mean success!
2090  **/
2091 static int
2092 i40e_update_filter_state(int count,
2093                          struct i40e_aqc_add_macvlan_element_data *add_list,
2094                          struct i40e_new_mac_filter *add_head)
2095 {
2096         int retval = 0;
2097         int i;
2098
2099         for (i = 0; i < count; i++) {
2100                 /* Always check status of each filter. We don't need to check
2101                  * the firmware return status because we pre-set the filter
2102                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2103                  * request to the adminq. Thus, if it no longer matches then
2104                  * we know the filter is active.
2105                  */
2106                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2107                         add_head->state = I40E_FILTER_FAILED;
2108                 } else {
2109                         add_head->state = I40E_FILTER_ACTIVE;
2110                         retval++;
2111                 }
2112
2113                 add_head = i40e_next_filter(add_head);
2114                 if (!add_head)
2115                         break;
2116         }
2117
2118         return retval;
2119 }
2120
2121 /**
2122  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2123  * @vsi: ptr to the VSI
2124  * @vsi_name: name to display in messages
2125  * @list: the list of filters to send to firmware
2126  * @num_del: the number of filters to delete
2127  * @retval: Set to -EIO on failure to delete
2128  *
2129  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2130  * *retval instead of a return value so that success does not force ret_val to
2131  * be set to 0. This ensures that a sequence of calls to this function
2132  * preserve the previous value of *retval on successful delete.
2133  */
2134 static
2135 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2136                           struct i40e_aqc_remove_macvlan_element_data *list,
2137                           int num_del, int *retval)
2138 {
2139         struct i40e_hw *hw = &vsi->back->hw;
2140         i40e_status aq_ret;
2141         int aq_err;
2142
2143         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2144         aq_err = hw->aq.asq_last_status;
2145
2146         /* Explicitly ignore and do not report when firmware returns ENOENT */
2147         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2148                 *retval = -EIO;
2149                 dev_info(&vsi->back->pdev->dev,
2150                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2151                          vsi_name, i40e_stat_str(hw, aq_ret),
2152                          i40e_aq_str(hw, aq_err));
2153         }
2154 }
2155
2156 /**
2157  * i40e_aqc_add_filters - Request firmware to add a set of filters
2158  * @vsi: ptr to the VSI
2159  * @vsi_name: name to display in messages
2160  * @list: the list of filters to send to firmware
2161  * @add_head: Position in the add hlist
2162  * @num_add: the number of filters to add
2163  *
2164  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2165  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2166  * space for more filters.
2167  */
2168 static
2169 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2170                           struct i40e_aqc_add_macvlan_element_data *list,
2171                           struct i40e_new_mac_filter *add_head,
2172                           int num_add)
2173 {
2174         struct i40e_hw *hw = &vsi->back->hw;
2175         int aq_err, fcnt;
2176
2177         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2178         aq_err = hw->aq.asq_last_status;
2179         fcnt = i40e_update_filter_state(num_add, list, add_head);
2180
2181         if (fcnt != num_add) {
2182                 if (vsi->type == I40E_VSI_MAIN) {
2183                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2184                         dev_warn(&vsi->back->pdev->dev,
2185                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2186                                  i40e_aq_str(hw, aq_err), vsi_name);
2187                 } else if (vsi->type == I40E_VSI_SRIOV ||
2188                            vsi->type == I40E_VSI_VMDQ1 ||
2189                            vsi->type == I40E_VSI_VMDQ2) {
2190                         dev_warn(&vsi->back->pdev->dev,
2191                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2192                                  i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2193                 } else {
2194                         dev_warn(&vsi->back->pdev->dev,
2195                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2196                                  i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2197                 }
2198         }
2199 }
2200
2201 /**
2202  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2203  * @vsi: pointer to the VSI
2204  * @vsi_name: the VSI name
2205  * @f: filter data
2206  *
2207  * This function sets or clears the promiscuous broadcast flags for VLAN
2208  * filters in order to properly receive broadcast frames. Assumes that only
2209  * broadcast filters are passed.
2210  *
2211  * Returns status indicating success or failure;
2212  **/
2213 static i40e_status
2214 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2215                           struct i40e_mac_filter *f)
2216 {
2217         bool enable = f->state == I40E_FILTER_NEW;
2218         struct i40e_hw *hw = &vsi->back->hw;
2219         i40e_status aq_ret;
2220
2221         if (f->vlan == I40E_VLAN_ANY) {
2222                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2223                                                    vsi->seid,
2224                                                    enable,
2225                                                    NULL);
2226         } else {
2227                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2228                                                             vsi->seid,
2229                                                             enable,
2230                                                             f->vlan,
2231                                                             NULL);
2232         }
2233
2234         if (aq_ret) {
2235                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2236                 dev_warn(&vsi->back->pdev->dev,
2237                          "Error %s, forcing overflow promiscuous on %s\n",
2238                          i40e_aq_str(hw, hw->aq.asq_last_status),
2239                          vsi_name);
2240         }
2241
2242         return aq_ret;
2243 }
2244
2245 /**
2246  * i40e_set_promiscuous - set promiscuous mode
2247  * @pf: board private structure
2248  * @promisc: promisc on or off
2249  *
2250  * There are different ways of setting promiscuous mode on a PF depending on
2251  * what state/environment we're in.  This identifies and sets it appropriately.
2252  * Returns 0 on success.
2253  **/
2254 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2255 {
2256         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2257         struct i40e_hw *hw = &pf->hw;
2258         i40e_status aq_ret;
2259
2260         if (vsi->type == I40E_VSI_MAIN &&
2261             pf->lan_veb != I40E_NO_VEB &&
2262             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2263                 /* set defport ON for Main VSI instead of true promisc
2264                  * this way we will get all unicast/multicast and VLAN
2265                  * promisc behavior but will not get VF or VMDq traffic
2266                  * replicated on the Main VSI.
2267                  */
2268                 if (promisc)
2269                         aq_ret = i40e_aq_set_default_vsi(hw,
2270                                                          vsi->seid,
2271                                                          NULL);
2272                 else
2273                         aq_ret = i40e_aq_clear_default_vsi(hw,
2274                                                            vsi->seid,
2275                                                            NULL);
2276                 if (aq_ret) {
2277                         dev_info(&pf->pdev->dev,
2278                                  "Set default VSI failed, err %s, aq_err %s\n",
2279                                  i40e_stat_str(hw, aq_ret),
2280                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2281                 }
2282         } else {
2283                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2284                                                   hw,
2285                                                   vsi->seid,
2286                                                   promisc, NULL,
2287                                                   true);
2288                 if (aq_ret) {
2289                         dev_info(&pf->pdev->dev,
2290                                  "set unicast promisc failed, err %s, aq_err %s\n",
2291                                  i40e_stat_str(hw, aq_ret),
2292                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2293                 }
2294                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2295                                                   hw,
2296                                                   vsi->seid,
2297                                                   promisc, NULL);
2298                 if (aq_ret) {
2299                         dev_info(&pf->pdev->dev,
2300                                  "set multicast promisc failed, err %s, aq_err %s\n",
2301                                  i40e_stat_str(hw, aq_ret),
2302                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2303                 }
2304         }
2305
2306         if (!aq_ret)
2307                 pf->cur_promisc = promisc;
2308
2309         return aq_ret;
2310 }
2311
2312 /**
2313  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2314  * @vsi: ptr to the VSI
2315  *
2316  * Push any outstanding VSI filter changes through the AdminQ.
2317  *
2318  * Returns 0 or error value
2319  **/
2320 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2321 {
2322         struct hlist_head tmp_add_list, tmp_del_list;
2323         struct i40e_mac_filter *f;
2324         struct i40e_new_mac_filter *new, *add_head = NULL;
2325         struct i40e_hw *hw = &vsi->back->hw;
2326         bool old_overflow, new_overflow;
2327         unsigned int failed_filters = 0;
2328         unsigned int vlan_filters = 0;
2329         char vsi_name[16] = "PF";
2330         int filter_list_len = 0;
2331         i40e_status aq_ret = 0;
2332         u32 changed_flags = 0;
2333         struct hlist_node *h;
2334         struct i40e_pf *pf;
2335         int num_add = 0;
2336         int num_del = 0;
2337         int retval = 0;
2338         u16 cmd_flags;
2339         int list_size;
2340         int bkt;
2341
2342         /* empty array typed pointers, kcalloc later */
2343         struct i40e_aqc_add_macvlan_element_data *add_list;
2344         struct i40e_aqc_remove_macvlan_element_data *del_list;
2345
2346         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2347                 usleep_range(1000, 2000);
2348         pf = vsi->back;
2349
2350         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2351
2352         if (vsi->netdev) {
2353                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2354                 vsi->current_netdev_flags = vsi->netdev->flags;
2355         }
2356
2357         INIT_HLIST_HEAD(&tmp_add_list);
2358         INIT_HLIST_HEAD(&tmp_del_list);
2359
2360         if (vsi->type == I40E_VSI_SRIOV)
2361                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2362         else if (vsi->type != I40E_VSI_MAIN)
2363                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2364
2365         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2366                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2367
2368                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2369                 /* Create a list of filters to delete. */
2370                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2371                         if (f->state == I40E_FILTER_REMOVE) {
2372                                 /* Move the element into temporary del_list */
2373                                 hash_del(&f->hlist);
2374                                 hlist_add_head(&f->hlist, &tmp_del_list);
2375
2376                                 /* Avoid counting removed filters */
2377                                 continue;
2378                         }
2379                         if (f->state == I40E_FILTER_NEW) {
2380                                 /* Create a temporary i40e_new_mac_filter */
2381                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2382                                 if (!new)
2383                                         goto err_no_memory_locked;
2384
2385                                 /* Store pointer to the real filter */
2386                                 new->f = f;
2387                                 new->state = f->state;
2388
2389                                 /* Add it to the hash list */
2390                                 hlist_add_head(&new->hlist, &tmp_add_list);
2391                         }
2392
2393                         /* Count the number of active (current and new) VLAN
2394                          * filters we have now. Does not count filters which
2395                          * are marked for deletion.
2396                          */
2397                         if (f->vlan > 0)
2398                                 vlan_filters++;
2399                 }
2400
2401                 retval = i40e_correct_mac_vlan_filters(vsi,
2402                                                        &tmp_add_list,
2403                                                        &tmp_del_list,
2404                                                        vlan_filters);
2405
2406                 hlist_for_each_entry(new, &tmp_add_list, hlist)
2407                         netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2408
2409                 if (retval)
2410                         goto err_no_memory_locked;
2411
2412                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2413         }
2414
2415         /* Now process 'del_list' outside the lock */
2416         if (!hlist_empty(&tmp_del_list)) {
2417                 filter_list_len = hw->aq.asq_buf_size /
2418                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2419                 list_size = filter_list_len *
2420                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2421                 del_list = kzalloc(list_size, GFP_ATOMIC);
2422                 if (!del_list)
2423                         goto err_no_memory;
2424
2425                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2426                         cmd_flags = 0;
2427
2428                         /* handle broadcast filters by updating the broadcast
2429                          * promiscuous flag and release filter list.
2430                          */
2431                         if (is_broadcast_ether_addr(f->macaddr)) {
2432                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2433
2434                                 hlist_del(&f->hlist);
2435                                 kfree(f);
2436                                 continue;
2437                         }
2438
2439                         /* add to delete list */
2440                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2441                         if (f->vlan == I40E_VLAN_ANY) {
2442                                 del_list[num_del].vlan_tag = 0;
2443                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2444                         } else {
2445                                 del_list[num_del].vlan_tag =
2446                                         cpu_to_le16((u16)(f->vlan));
2447                         }
2448
2449                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2450                         del_list[num_del].flags = cmd_flags;
2451                         num_del++;
2452
2453                         /* flush a full buffer */
2454                         if (num_del == filter_list_len) {
2455                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2456                                                      num_del, &retval);
2457                                 memset(del_list, 0, list_size);
2458                                 num_del = 0;
2459                         }
2460                         /* Release memory for MAC filter entries which were
2461                          * synced up with HW.
2462                          */
2463                         hlist_del(&f->hlist);
2464                         kfree(f);
2465                 }
2466
2467                 if (num_del) {
2468                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2469                                              num_del, &retval);
2470                 }
2471
2472                 kfree(del_list);
2473                 del_list = NULL;
2474         }
2475
2476         if (!hlist_empty(&tmp_add_list)) {
2477                 /* Do all the adds now. */
2478                 filter_list_len = hw->aq.asq_buf_size /
2479                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2480                 list_size = filter_list_len *
2481                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2482                 add_list = kzalloc(list_size, GFP_ATOMIC);
2483                 if (!add_list)
2484                         goto err_no_memory;
2485
2486                 num_add = 0;
2487                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2488                         /* handle broadcast filters by updating the broadcast
2489                          * promiscuous flag instead of adding a MAC filter.
2490                          */
2491                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2492                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2493                                                               new->f))
2494                                         new->state = I40E_FILTER_FAILED;
2495                                 else
2496                                         new->state = I40E_FILTER_ACTIVE;
2497                                 continue;
2498                         }
2499
2500                         /* add to add array */
2501                         if (num_add == 0)
2502                                 add_head = new;
2503                         cmd_flags = 0;
2504                         ether_addr_copy(add_list[num_add].mac_addr,
2505                                         new->f->macaddr);
2506                         if (new->f->vlan == I40E_VLAN_ANY) {
2507                                 add_list[num_add].vlan_tag = 0;
2508                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2509                         } else {
2510                                 add_list[num_add].vlan_tag =
2511                                         cpu_to_le16((u16)(new->f->vlan));
2512                         }
2513                         add_list[num_add].queue_number = 0;
2514                         /* set invalid match method for later detection */
2515                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2516                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2517                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2518                         num_add++;
2519
2520                         /* flush a full buffer */
2521                         if (num_add == filter_list_len) {
2522                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2523                                                      add_head, num_add);
2524                                 memset(add_list, 0, list_size);
2525                                 num_add = 0;
2526                         }
2527                 }
2528                 if (num_add) {
2529                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2530                                              num_add);
2531                 }
2532                 /* Now move all of the filters from the temp add list back to
2533                  * the VSI's list.
2534                  */
2535                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2536                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2537                         /* Only update the state if we're still NEW */
2538                         if (new->f->state == I40E_FILTER_NEW)
2539                                 new->f->state = new->state;
2540                         hlist_del(&new->hlist);
2541                         netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2542                         kfree(new);
2543                 }
2544                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2545                 kfree(add_list);
2546                 add_list = NULL;
2547         }
2548
2549         /* Determine the number of active and failed filters. */
2550         spin_lock_bh(&vsi->mac_filter_hash_lock);
2551         vsi->active_filters = 0;
2552         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2553                 if (f->state == I40E_FILTER_ACTIVE)
2554                         vsi->active_filters++;
2555                 else if (f->state == I40E_FILTER_FAILED)
2556                         failed_filters++;
2557         }
2558         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2559
2560         /* Check if we are able to exit overflow promiscuous mode. We can
2561          * safely exit if we didn't just enter, we no longer have any failed
2562          * filters, and we have reduced filters below the threshold value.
2563          */
2564         if (old_overflow && !failed_filters &&
2565             vsi->active_filters < vsi->promisc_threshold) {
2566                 dev_info(&pf->pdev->dev,
2567                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2568                          vsi_name);
2569                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2570                 vsi->promisc_threshold = 0;
2571         }
2572
2573         /* if the VF is not trusted do not do promisc */
2574         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2575                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2576                 goto out;
2577         }
2578
2579         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2580
2581         /* If we are entering overflow promiscuous, we need to calculate a new
2582          * threshold for when we are safe to exit
2583          */
2584         if (!old_overflow && new_overflow)
2585                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2586
2587         /* check for changes in promiscuous modes */
2588         if (changed_flags & IFF_ALLMULTI) {
2589                 bool cur_multipromisc;
2590
2591                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2592                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2593                                                                vsi->seid,
2594                                                                cur_multipromisc,
2595                                                                NULL);
2596                 if (aq_ret) {
2597                         retval = i40e_aq_rc_to_posix(aq_ret,
2598                                                      hw->aq.asq_last_status);
2599                         dev_info(&pf->pdev->dev,
2600                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2601                                  vsi_name,
2602                                  i40e_stat_str(hw, aq_ret),
2603                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2604                 } else {
2605                         dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2606                                  cur_multipromisc ? "entering" : "leaving");
2607                 }
2608         }
2609
2610         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2611                 bool cur_promisc;
2612
2613                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2614                                new_overflow);
2615                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2616                 if (aq_ret) {
2617                         retval = i40e_aq_rc_to_posix(aq_ret,
2618                                                      hw->aq.asq_last_status);
2619                         dev_info(&pf->pdev->dev,
2620                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2621                                  cur_promisc ? "on" : "off",
2622                                  vsi_name,
2623                                  i40e_stat_str(hw, aq_ret),
2624                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2625                 }
2626         }
2627 out:
2628         /* if something went wrong then set the changed flag so we try again */
2629         if (retval)
2630                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2631
2632         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2633         return retval;
2634
2635 err_no_memory:
2636         /* Restore elements on the temporary add and delete lists */
2637         spin_lock_bh(&vsi->mac_filter_hash_lock);
2638 err_no_memory_locked:
2639         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2640         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2641         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2642
2643         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2644         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2645         return -ENOMEM;
2646 }
2647
2648 /**
2649  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2650  * @pf: board private structure
2651  **/
2652 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2653 {
2654         int v;
2655
2656         if (!pf)
2657                 return;
2658         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2659                 return;
2660         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2661                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2662                 return;
2663         }
2664
2665         for (v = 0; v < pf->num_alloc_vsi; v++) {
2666                 if (pf->vsi[v] &&
2667                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2668                     !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2669                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2670
2671                         if (ret) {
2672                                 /* come back and try again later */
2673                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2674                                         pf->state);
2675                                 break;
2676                         }
2677                 }
2678         }
2679 }
2680
2681 /**
2682  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2683  * @vsi: the vsi
2684  **/
2685 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2686 {
2687         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2688                 return I40E_RXBUFFER_2048;
2689         else
2690                 return I40E_RXBUFFER_3072;
2691 }
2692
2693 /**
2694  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2695  * @netdev: network interface device structure
2696  * @new_mtu: new value for maximum frame size
2697  *
2698  * Returns 0 on success, negative on failure
2699  **/
2700 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2701 {
2702         struct i40e_netdev_priv *np = netdev_priv(netdev);
2703         struct i40e_vsi *vsi = np->vsi;
2704         struct i40e_pf *pf = vsi->back;
2705
2706         if (i40e_enabled_xdp_vsi(vsi)) {
2707                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2708
2709                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2710                         return -EINVAL;
2711         }
2712
2713         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2714                    netdev->mtu, new_mtu);
2715         netdev->mtu = new_mtu;
2716         if (netif_running(netdev))
2717                 i40e_vsi_reinit_locked(vsi);
2718         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2719         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2720         return 0;
2721 }
2722
2723 /**
2724  * i40e_ioctl - Access the hwtstamp interface
2725  * @netdev: network interface device structure
2726  * @ifr: interface request data
2727  * @cmd: ioctl command
2728  **/
2729 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2730 {
2731         struct i40e_netdev_priv *np = netdev_priv(netdev);
2732         struct i40e_pf *pf = np->vsi->back;
2733
2734         switch (cmd) {
2735         case SIOCGHWTSTAMP:
2736                 return i40e_ptp_get_ts_config(pf, ifr);
2737         case SIOCSHWTSTAMP:
2738                 return i40e_ptp_set_ts_config(pf, ifr);
2739         default:
2740                 return -EOPNOTSUPP;
2741         }
2742 }
2743
2744 /**
2745  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2746  * @vsi: the vsi being adjusted
2747  **/
2748 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2749 {
2750         struct i40e_vsi_context ctxt;
2751         i40e_status ret;
2752
2753         /* Don't modify stripping options if a port VLAN is active */
2754         if (vsi->info.pvid)
2755                 return;
2756
2757         if ((vsi->info.valid_sections &
2758              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2759             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2760                 return;  /* already enabled */
2761
2762         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2763         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2764                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2765
2766         ctxt.seid = vsi->seid;
2767         ctxt.info = vsi->info;
2768         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2769         if (ret) {
2770                 dev_info(&vsi->back->pdev->dev,
2771                          "update vlan stripping failed, err %s aq_err %s\n",
2772                          i40e_stat_str(&vsi->back->hw, ret),
2773                          i40e_aq_str(&vsi->back->hw,
2774                                      vsi->back->hw.aq.asq_last_status));
2775         }
2776 }
2777
2778 /**
2779  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2780  * @vsi: the vsi being adjusted
2781  **/
2782 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2783 {
2784         struct i40e_vsi_context ctxt;
2785         i40e_status ret;
2786
2787         /* Don't modify stripping options if a port VLAN is active */
2788         if (vsi->info.pvid)
2789                 return;
2790
2791         if ((vsi->info.valid_sections &
2792              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2793             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2794              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2795                 return;  /* already disabled */
2796
2797         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2798         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2799                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2800
2801         ctxt.seid = vsi->seid;
2802         ctxt.info = vsi->info;
2803         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2804         if (ret) {
2805                 dev_info(&vsi->back->pdev->dev,
2806                          "update vlan stripping failed, err %s aq_err %s\n",
2807                          i40e_stat_str(&vsi->back->hw, ret),
2808                          i40e_aq_str(&vsi->back->hw,
2809                                      vsi->back->hw.aq.asq_last_status));
2810         }
2811 }
2812
2813 /**
2814  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2815  * @vsi: the vsi being configured
2816  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2817  *
2818  * This is a helper function for adding a new MAC/VLAN filter with the
2819  * specified VLAN for each existing MAC address already in the hash table.
2820  * This function does *not* perform any accounting to update filters based on
2821  * VLAN mode.
2822  *
2823  * NOTE: this function expects to be called while under the
2824  * mac_filter_hash_lock
2825  **/
2826 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2827 {
2828         struct i40e_mac_filter *f, *add_f;
2829         struct hlist_node *h;
2830         int bkt;
2831
2832         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2833                 if (f->state == I40E_FILTER_REMOVE)
2834                         continue;
2835                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2836                 if (!add_f) {
2837                         dev_info(&vsi->back->pdev->dev,
2838                                  "Could not add vlan filter %d for %pM\n",
2839                                  vid, f->macaddr);
2840                         return -ENOMEM;
2841                 }
2842         }
2843
2844         return 0;
2845 }
2846
2847 /**
2848  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2849  * @vsi: the VSI being configured
2850  * @vid: VLAN id to be added
2851  **/
2852 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2853 {
2854         int err;
2855
2856         if (vsi->info.pvid)
2857                 return -EINVAL;
2858
2859         /* The network stack will attempt to add VID=0, with the intention to
2860          * receive priority tagged packets with a VLAN of 0. Our HW receives
2861          * these packets by default when configured to receive untagged
2862          * packets, so we don't need to add a filter for this case.
2863          * Additionally, HW interprets adding a VID=0 filter as meaning to
2864          * receive *only* tagged traffic and stops receiving untagged traffic.
2865          * Thus, we do not want to actually add a filter for VID=0
2866          */
2867         if (!vid)
2868                 return 0;
2869
2870         /* Locked once because all functions invoked below iterates list*/
2871         spin_lock_bh(&vsi->mac_filter_hash_lock);
2872         err = i40e_add_vlan_all_mac(vsi, vid);
2873         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2874         if (err)
2875                 return err;
2876
2877         /* schedule our worker thread which will take care of
2878          * applying the new filter changes
2879          */
2880         i40e_service_event_schedule(vsi->back);
2881         return 0;
2882 }
2883
2884 /**
2885  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2886  * @vsi: the vsi being configured
2887  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2888  *
2889  * This function should be used to remove all VLAN filters which match the
2890  * given VID. It does not schedule the service event and does not take the
2891  * mac_filter_hash_lock so it may be combined with other operations under
2892  * a single invocation of the mac_filter_hash_lock.
2893  *
2894  * NOTE: this function expects to be called while under the
2895  * mac_filter_hash_lock
2896  */
2897 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2898 {
2899         struct i40e_mac_filter *f;
2900         struct hlist_node *h;
2901         int bkt;
2902
2903         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2904                 if (f->vlan == vid)
2905                         __i40e_del_filter(vsi, f);
2906         }
2907 }
2908
2909 /**
2910  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2911  * @vsi: the VSI being configured
2912  * @vid: VLAN id to be removed
2913  **/
2914 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2915 {
2916         if (!vid || vsi->info.pvid)
2917                 return;
2918
2919         spin_lock_bh(&vsi->mac_filter_hash_lock);
2920         i40e_rm_vlan_all_mac(vsi, vid);
2921         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2922
2923         /* schedule our worker thread which will take care of
2924          * applying the new filter changes
2925          */
2926         i40e_service_event_schedule(vsi->back);
2927 }
2928
2929 /**
2930  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2931  * @netdev: network interface to be adjusted
2932  * @proto: unused protocol value
2933  * @vid: vlan id to be added
2934  *
2935  * net_device_ops implementation for adding vlan ids
2936  **/
2937 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2938                                 __always_unused __be16 proto, u16 vid)
2939 {
2940         struct i40e_netdev_priv *np = netdev_priv(netdev);
2941         struct i40e_vsi *vsi = np->vsi;
2942         int ret = 0;
2943
2944         if (vid >= VLAN_N_VID)
2945                 return -EINVAL;
2946
2947         ret = i40e_vsi_add_vlan(vsi, vid);
2948         if (!ret)
2949                 set_bit(vid, vsi->active_vlans);
2950
2951         return ret;
2952 }
2953
2954 /**
2955  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
2956  * @netdev: network interface to be adjusted
2957  * @proto: unused protocol value
2958  * @vid: vlan id to be added
2959  **/
2960 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2961                                     __always_unused __be16 proto, u16 vid)
2962 {
2963         struct i40e_netdev_priv *np = netdev_priv(netdev);
2964         struct i40e_vsi *vsi = np->vsi;
2965
2966         if (vid >= VLAN_N_VID)
2967                 return;
2968         set_bit(vid, vsi->active_vlans);
2969 }
2970
2971 /**
2972  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2973  * @netdev: network interface to be adjusted
2974  * @proto: unused protocol value
2975  * @vid: vlan id to be removed
2976  *
2977  * net_device_ops implementation for removing vlan ids
2978  **/
2979 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2980                                  __always_unused __be16 proto, u16 vid)
2981 {
2982         struct i40e_netdev_priv *np = netdev_priv(netdev);
2983         struct i40e_vsi *vsi = np->vsi;
2984
2985         /* return code is ignored as there is nothing a user
2986          * can do about failure to remove and a log message was
2987          * already printed from the other function
2988          */
2989         i40e_vsi_kill_vlan(vsi, vid);
2990
2991         clear_bit(vid, vsi->active_vlans);
2992
2993         return 0;
2994 }
2995
2996 /**
2997  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2998  * @vsi: the vsi being brought back up
2999  **/
3000 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3001 {
3002         u16 vid;
3003
3004         if (!vsi->netdev)
3005                 return;
3006
3007         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3008                 i40e_vlan_stripping_enable(vsi);
3009         else
3010                 i40e_vlan_stripping_disable(vsi);
3011
3012         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3013                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3014                                         vid);
3015 }
3016
3017 /**
3018  * i40e_vsi_add_pvid - Add pvid for the VSI
3019  * @vsi: the vsi being adjusted
3020  * @vid: the vlan id to set as a PVID
3021  **/
3022 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3023 {
3024         struct i40e_vsi_context ctxt;
3025         i40e_status ret;
3026
3027         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3028         vsi->info.pvid = cpu_to_le16(vid);
3029         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3030                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
3031                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
3032
3033         ctxt.seid = vsi->seid;
3034         ctxt.info = vsi->info;
3035         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3036         if (ret) {
3037                 dev_info(&vsi->back->pdev->dev,
3038                          "add pvid failed, err %s aq_err %s\n",
3039                          i40e_stat_str(&vsi->back->hw, ret),
3040                          i40e_aq_str(&vsi->back->hw,
3041                                      vsi->back->hw.aq.asq_last_status));
3042                 return -ENOENT;
3043         }
3044
3045         return 0;
3046 }
3047
3048 /**
3049  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3050  * @vsi: the vsi being adjusted
3051  *
3052  * Just use the vlan_rx_register() service to put it back to normal
3053  **/
3054 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3055 {
3056         vsi->info.pvid = 0;
3057
3058         i40e_vlan_stripping_disable(vsi);
3059 }
3060
3061 /**
3062  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3063  * @vsi: ptr to the VSI
3064  *
3065  * If this function returns with an error, then it's possible one or
3066  * more of the rings is populated (while the rest are not).  It is the
3067  * callers duty to clean those orphaned rings.
3068  *
3069  * Return 0 on success, negative on failure
3070  **/
3071 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3072 {
3073         int i, err = 0;
3074
3075         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3076                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3077
3078         if (!i40e_enabled_xdp_vsi(vsi))
3079                 return err;
3080
3081         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3082                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3083
3084         return err;
3085 }
3086
3087 /**
3088  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3089  * @vsi: ptr to the VSI
3090  *
3091  * Free VSI's transmit software resources
3092  **/
3093 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3094 {
3095         int i;
3096
3097         if (vsi->tx_rings) {
3098                 for (i = 0; i < vsi->num_queue_pairs; i++)
3099                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3100                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3101         }
3102
3103         if (vsi->xdp_rings) {
3104                 for (i = 0; i < vsi->num_queue_pairs; i++)
3105                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3106                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3107         }
3108 }
3109
3110 /**
3111  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3112  * @vsi: ptr to the VSI
3113  *
3114  * If this function returns with an error, then it's possible one or
3115  * more of the rings is populated (while the rest are not).  It is the
3116  * callers duty to clean those orphaned rings.
3117  *
3118  * Return 0 on success, negative on failure
3119  **/
3120 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3121 {
3122         int i, err = 0;
3123
3124         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3125                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3126         return err;
3127 }
3128
3129 /**
3130  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3131  * @vsi: ptr to the VSI
3132  *
3133  * Free all receive software resources
3134  **/
3135 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3136 {
3137         int i;
3138
3139         if (!vsi->rx_rings)
3140                 return;
3141
3142         for (i = 0; i < vsi->num_queue_pairs; i++)
3143                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3144                         i40e_free_rx_resources(vsi->rx_rings[i]);
3145 }
3146
3147 /**
3148  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3149  * @ring: The Tx ring to configure
3150  *
3151  * This enables/disables XPS for a given Tx descriptor ring
3152  * based on the TCs enabled for the VSI that ring belongs to.
3153  **/
3154 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3155 {
3156         int cpu;
3157
3158         if (!ring->q_vector || !ring->netdev || ring->ch)
3159                 return;
3160
3161         /* We only initialize XPS once, so as not to overwrite user settings */
3162         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3163                 return;
3164
3165         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3166         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3167                             ring->queue_index);
3168 }
3169
3170 /**
3171  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3172  * @ring: The Tx or Rx ring
3173  *
3174  * Returns the AF_XDP buffer pool or NULL.
3175  **/
3176 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3177 {
3178         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3179         int qid = ring->queue_index;
3180
3181         if (ring_is_xdp(ring))
3182                 qid -= ring->vsi->alloc_queue_pairs;
3183
3184         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3185                 return NULL;
3186
3187         return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3188 }
3189
3190 /**
3191  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3192  * @ring: The Tx ring to configure
3193  *
3194  * Configure the Tx descriptor ring in the HMC context.
3195  **/
3196 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3197 {
3198         struct i40e_vsi *vsi = ring->vsi;
3199         u16 pf_q = vsi->base_queue + ring->queue_index;
3200         struct i40e_hw *hw = &vsi->back->hw;
3201         struct i40e_hmc_obj_txq tx_ctx;
3202         i40e_status err = 0;
3203         u32 qtx_ctl = 0;
3204
3205         if (ring_is_xdp(ring))
3206                 ring->xsk_pool = i40e_xsk_pool(ring);
3207
3208         /* some ATR related tx ring init */
3209         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3210                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3211                 ring->atr_count = 0;
3212         } else {
3213                 ring->atr_sample_rate = 0;
3214         }
3215
3216         /* configure XPS */
3217         i40e_config_xps_tx_ring(ring);
3218
3219         /* clear the context structure first */
3220         memset(&tx_ctx, 0, sizeof(tx_ctx));
3221
3222         tx_ctx.new_context = 1;
3223         tx_ctx.base = (ring->dma / 128);
3224         tx_ctx.qlen = ring->count;
3225         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3226                                                I40E_FLAG_FD_ATR_ENABLED));
3227         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3228         /* FDIR VSI tx ring can still use RS bit and writebacks */
3229         if (vsi->type != I40E_VSI_FDIR)
3230                 tx_ctx.head_wb_ena = 1;
3231         tx_ctx.head_wb_addr = ring->dma +
3232                               (ring->count * sizeof(struct i40e_tx_desc));
3233
3234         /* As part of VSI creation/update, FW allocates certain
3235          * Tx arbitration queue sets for each TC enabled for
3236          * the VSI. The FW returns the handles to these queue
3237          * sets as part of the response buffer to Add VSI,
3238          * Update VSI, etc. AQ commands. It is expected that
3239          * these queue set handles be associated with the Tx
3240          * queues by the driver as part of the TX queue context
3241          * initialization. This has to be done regardless of
3242          * DCB as by default everything is mapped to TC0.
3243          */
3244
3245         if (ring->ch)
3246                 tx_ctx.rdylist =
3247                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3248
3249         else
3250                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3251
3252         tx_ctx.rdylist_act = 0;
3253
3254         /* clear the context in the HMC */
3255         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3256         if (err) {
3257                 dev_info(&vsi->back->pdev->dev,
3258                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3259                          ring->queue_index, pf_q, err);
3260                 return -ENOMEM;
3261         }
3262
3263         /* set the context in the HMC */
3264         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3265         if (err) {
3266                 dev_info(&vsi->back->pdev->dev,
3267                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3268                          ring->queue_index, pf_q, err);
3269                 return -ENOMEM;
3270         }
3271
3272         /* Now associate this queue with this PCI function */
3273         if (ring->ch) {
3274                 if (ring->ch->type == I40E_VSI_VMDQ2)
3275                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3276                 else
3277                         return -EINVAL;
3278
3279                 qtx_ctl |= (ring->ch->vsi_number <<
3280                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3281                             I40E_QTX_CTL_VFVM_INDX_MASK;
3282         } else {
3283                 if (vsi->type == I40E_VSI_VMDQ2) {
3284                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3285                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3286                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3287                 } else {
3288                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3289                 }
3290         }
3291
3292         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3293                     I40E_QTX_CTL_PF_INDX_MASK);
3294         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3295         i40e_flush(hw);
3296
3297         /* cache tail off for easier writes later */
3298         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3299
3300         return 0;
3301 }
3302
3303 /**
3304  * i40e_rx_offset - Return expected offset into page to access data
3305  * @rx_ring: Ring we are requesting offset of
3306  *
3307  * Returns the offset value for ring into the data buffer.
3308  */
3309 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3310 {
3311         return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3312 }
3313
3314 /**
3315  * i40e_configure_rx_ring - Configure a receive ring context
3316  * @ring: The Rx ring to configure
3317  *
3318  * Configure the Rx descriptor ring in the HMC context.
3319  **/
3320 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3321 {
3322         struct i40e_vsi *vsi = ring->vsi;
3323         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3324         u16 pf_q = vsi->base_queue + ring->queue_index;
3325         struct i40e_hw *hw = &vsi->back->hw;
3326         struct i40e_hmc_obj_rxq rx_ctx;
3327         i40e_status err = 0;
3328         bool ok;
3329         int ret;
3330
3331         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3332
3333         /* clear the context structure first */
3334         memset(&rx_ctx, 0, sizeof(rx_ctx));
3335
3336         if (ring->vsi->type == I40E_VSI_MAIN)
3337                 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3338
3339         kfree(ring->rx_bi);
3340         ring->xsk_pool = i40e_xsk_pool(ring);
3341         if (ring->xsk_pool) {
3342                 ret = i40e_alloc_rx_bi_zc(ring);
3343                 if (ret)
3344                         return ret;
3345                 ring->rx_buf_len =
3346                   xsk_pool_get_rx_frame_size(ring->xsk_pool);
3347                 /* For AF_XDP ZC, we disallow packets to span on
3348                  * multiple buffers, thus letting us skip that
3349                  * handling in the fast-path.
3350                  */
3351                 chain_len = 1;
3352                 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3353                                                  MEM_TYPE_XSK_BUFF_POOL,
3354                                                  NULL);
3355                 if (ret)
3356                         return ret;
3357                 dev_info(&vsi->back->pdev->dev,
3358                          "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3359                          ring->queue_index);
3360
3361         } else {
3362                 ret = i40e_alloc_rx_bi(ring);
3363                 if (ret)
3364                         return ret;
3365                 ring->rx_buf_len = vsi->rx_buf_len;
3366                 if (ring->vsi->type == I40E_VSI_MAIN) {
3367                         ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3368                                                          MEM_TYPE_PAGE_SHARED,
3369                                                          NULL);
3370                         if (ret)
3371                                 return ret;
3372                 }
3373         }
3374
3375         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3376                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3377
3378         rx_ctx.base = (ring->dma / 128);
3379         rx_ctx.qlen = ring->count;
3380
3381         /* use 16 byte descriptors */
3382         rx_ctx.dsize = 0;
3383
3384         /* descriptor type is always zero
3385          * rx_ctx.dtype = 0;
3386          */
3387         rx_ctx.hsplit_0 = 0;
3388
3389         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3390         if (hw->revision_id == 0)
3391                 rx_ctx.lrxqthresh = 0;
3392         else
3393                 rx_ctx.lrxqthresh = 1;
3394         rx_ctx.crcstrip = 1;
3395         rx_ctx.l2tsel = 1;
3396         /* this controls whether VLAN is stripped from inner headers */
3397         rx_ctx.showiv = 0;
3398         /* set the prefena field to 1 because the manual says to */
3399         rx_ctx.prefena = 1;
3400
3401         /* clear the context in the HMC */
3402         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3403         if (err) {
3404                 dev_info(&vsi->back->pdev->dev,
3405                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3406                          ring->queue_index, pf_q, err);
3407                 return -ENOMEM;
3408         }
3409
3410         /* set the context in the HMC */
3411         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3412         if (err) {
3413                 dev_info(&vsi->back->pdev->dev,
3414                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3415                          ring->queue_index, pf_q, err);
3416                 return -ENOMEM;
3417         }
3418
3419         /* configure Rx buffer alignment */
3420         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3421                 clear_ring_build_skb_enabled(ring);
3422         else
3423                 set_ring_build_skb_enabled(ring);
3424
3425         ring->rx_offset = i40e_rx_offset(ring);
3426
3427         /* cache tail for quicker writes, and clear the reg before use */
3428         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3429         writel(0, ring->tail);
3430
3431         if (ring->xsk_pool) {
3432                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3433                 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3434         } else {
3435                 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3436         }
3437         if (!ok) {
3438                 /* Log this in case the user has forgotten to give the kernel
3439                  * any buffers, even later in the application.
3440                  */
3441                 dev_info(&vsi->back->pdev->dev,
3442                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3443                          ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3444                          ring->queue_index, pf_q);
3445         }
3446
3447         return 0;
3448 }
3449
3450 /**
3451  * i40e_vsi_configure_tx - Configure the VSI for Tx
3452  * @vsi: VSI structure describing this set of rings and resources
3453  *
3454  * Configure the Tx VSI for operation.
3455  **/
3456 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3457 {
3458         int err = 0;
3459         u16 i;
3460
3461         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3462                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3463
3464         if (err || !i40e_enabled_xdp_vsi(vsi))
3465                 return err;
3466
3467         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3468                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3469
3470         return err;
3471 }
3472
3473 /**
3474  * i40e_vsi_configure_rx - Configure the VSI for Rx
3475  * @vsi: the VSI being configured
3476  *
3477  * Configure the Rx VSI for operation.
3478  **/
3479 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3480 {
3481         int err = 0;
3482         u16 i;
3483
3484         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3485                 vsi->max_frame = I40E_MAX_RXBUFFER;
3486                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3487 #if (PAGE_SIZE < 8192)
3488         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3489                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3490                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3491                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3492 #endif
3493         } else {
3494                 vsi->max_frame = I40E_MAX_RXBUFFER;
3495                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3496                                                        I40E_RXBUFFER_2048;
3497         }
3498
3499         /* set up individual rings */
3500         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3501                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3502
3503         return err;
3504 }
3505
3506 /**
3507  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3508  * @vsi: ptr to the VSI
3509  **/
3510 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3511 {
3512         struct i40e_ring *tx_ring, *rx_ring;
3513         u16 qoffset, qcount;
3514         int i, n;
3515
3516         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3517                 /* Reset the TC information */
3518                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3519                         rx_ring = vsi->rx_rings[i];
3520                         tx_ring = vsi->tx_rings[i];
3521                         rx_ring->dcb_tc = 0;
3522                         tx_ring->dcb_tc = 0;
3523                 }
3524                 return;
3525         }
3526
3527         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3528                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3529                         continue;
3530
3531                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3532                 qcount = vsi->tc_config.tc_info[n].qcount;
3533                 for (i = qoffset; i < (qoffset + qcount); i++) {
3534                         rx_ring = vsi->rx_rings[i];
3535                         tx_ring = vsi->tx_rings[i];
3536                         rx_ring->dcb_tc = n;
3537                         tx_ring->dcb_tc = n;
3538                 }
3539         }
3540 }
3541
3542 /**
3543  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3544  * @vsi: ptr to the VSI
3545  **/
3546 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3547 {
3548         if (vsi->netdev)
3549                 i40e_set_rx_mode(vsi->netdev);
3550 }
3551
3552 /**
3553  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3554  * @pf: Pointer to the targeted PF
3555  *
3556  * Set all flow director counters to 0.
3557  */
3558 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3559 {
3560         pf->fd_tcp4_filter_cnt = 0;
3561         pf->fd_udp4_filter_cnt = 0;
3562         pf->fd_sctp4_filter_cnt = 0;
3563         pf->fd_ip4_filter_cnt = 0;
3564         pf->fd_tcp6_filter_cnt = 0;
3565         pf->fd_udp6_filter_cnt = 0;
3566         pf->fd_sctp6_filter_cnt = 0;
3567         pf->fd_ip6_filter_cnt = 0;
3568 }
3569
3570 /**
3571  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3572  * @vsi: Pointer to the targeted VSI
3573  *
3574  * This function replays the hlist on the hw where all the SB Flow Director
3575  * filters were saved.
3576  **/
3577 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3578 {
3579         struct i40e_fdir_filter *filter;
3580         struct i40e_pf *pf = vsi->back;
3581         struct hlist_node *node;
3582
3583         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3584                 return;
3585
3586         /* Reset FDir counters as we're replaying all existing filters */
3587         i40e_reset_fdir_filter_cnt(pf);
3588
3589         hlist_for_each_entry_safe(filter, node,
3590                                   &pf->fdir_filter_list, fdir_node) {
3591                 i40e_add_del_fdir(vsi, filter, true);
3592         }
3593 }
3594
3595 /**
3596  * i40e_vsi_configure - Set up the VSI for action
3597  * @vsi: the VSI being configured
3598  **/
3599 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3600 {
3601         int err;
3602
3603         i40e_set_vsi_rx_mode(vsi);
3604         i40e_restore_vlan(vsi);
3605         i40e_vsi_config_dcb_rings(vsi);
3606         err = i40e_vsi_configure_tx(vsi);
3607         if (!err)
3608                 err = i40e_vsi_configure_rx(vsi);
3609
3610         return err;
3611 }
3612
3613 /**
3614  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3615  * @vsi: the VSI being configured
3616  **/
3617 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3618 {
3619         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3620         struct i40e_pf *pf = vsi->back;
3621         struct i40e_hw *hw = &pf->hw;
3622         u16 vector;
3623         int i, q;
3624         u32 qp;
3625
3626         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3627          * and PFINT_LNKLSTn registers, e.g.:
3628          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3629          */
3630         qp = vsi->base_queue;
3631         vector = vsi->base_vector;
3632         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3633                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3634
3635                 q_vector->rx.next_update = jiffies + 1;
3636                 q_vector->rx.target_itr =
3637                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3638                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3639                      q_vector->rx.target_itr >> 1);
3640                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3641
3642                 q_vector->tx.next_update = jiffies + 1;
3643                 q_vector->tx.target_itr =
3644                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3645                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3646                      q_vector->tx.target_itr >> 1);
3647                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3648
3649                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3650                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3651
3652                 /* Linked list for the queuepairs assigned to this vector */
3653                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3654                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3655                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3656                         u32 val;
3657
3658                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3659                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3660                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3661                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3662                               (I40E_QUEUE_TYPE_TX <<
3663                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3664
3665                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3666
3667                         if (has_xdp) {
3668                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3669                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3670                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3671                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3672                                       (I40E_QUEUE_TYPE_TX <<
3673                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3674
3675                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3676                         }
3677
3678                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3679                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3680                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3681                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3682                               (I40E_QUEUE_TYPE_RX <<
3683                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3684
3685                         /* Terminate the linked list */
3686                         if (q == (q_vector->num_ringpairs - 1))
3687                                 val |= (I40E_QUEUE_END_OF_LIST <<
3688                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3689
3690                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3691                         qp++;
3692                 }
3693         }
3694
3695         i40e_flush(hw);
3696 }
3697
3698 /**
3699  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3700  * @pf: pointer to private device data structure
3701  **/
3702 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3703 {
3704         struct i40e_hw *hw = &pf->hw;
3705         u32 val;
3706
3707         /* clear things first */
3708         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3709         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3710
3711         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3712               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3713               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3714               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3715               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3716               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3717               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3718               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3719
3720         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3721                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3722
3723         if (pf->flags & I40E_FLAG_PTP)
3724                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3725
3726         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3727
3728         /* SW_ITR_IDX = 0, but don't change INTENA */
3729         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3730                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3731
3732         /* OTHER_ITR_IDX = 0 */
3733         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3734 }
3735
3736 /**
3737  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3738  * @vsi: the VSI being configured
3739  **/
3740 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3741 {
3742         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3743         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3744         struct i40e_pf *pf = vsi->back;
3745         struct i40e_hw *hw = &pf->hw;
3746         u32 val;
3747
3748         /* set the ITR configuration */
3749         q_vector->rx.next_update = jiffies + 1;
3750         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3751         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3752         q_vector->rx.current_itr = q_vector->rx.target_itr;
3753         q_vector->tx.next_update = jiffies + 1;
3754         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3755         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3756         q_vector->tx.current_itr = q_vector->tx.target_itr;
3757
3758         i40e_enable_misc_int_causes(pf);
3759
3760         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3761         wr32(hw, I40E_PFINT_LNKLST0, 0);
3762
3763         /* Associate the queue pair to the vector and enable the queue int */
3764         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3765               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3766               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3767               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3768
3769         wr32(hw, I40E_QINT_RQCTL(0), val);
3770
3771         if (i40e_enabled_xdp_vsi(vsi)) {
3772                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3773                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3774                       (I40E_QUEUE_TYPE_TX
3775                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3776
3777                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3778         }
3779
3780         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3781               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3782               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3783
3784         wr32(hw, I40E_QINT_TQCTL(0), val);
3785         i40e_flush(hw);
3786 }
3787
3788 /**
3789  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3790  * @pf: board private structure
3791  **/
3792 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3793 {
3794         struct i40e_hw *hw = &pf->hw;
3795
3796         wr32(hw, I40E_PFINT_DYN_CTL0,
3797              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3798         i40e_flush(hw);
3799 }
3800
3801 /**
3802  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3803  * @pf: board private structure
3804  **/
3805 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3806 {
3807         struct i40e_hw *hw = &pf->hw;
3808         u32 val;
3809
3810         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3811               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3812               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3813
3814         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3815         i40e_flush(hw);
3816 }
3817
3818 /**
3819  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3820  * @irq: interrupt number
3821  * @data: pointer to a q_vector
3822  **/
3823 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3824 {
3825         struct i40e_q_vector *q_vector = data;
3826
3827         if (!q_vector->tx.ring && !q_vector->rx.ring)
3828                 return IRQ_HANDLED;
3829
3830         napi_schedule_irqoff(&q_vector->napi);
3831
3832         return IRQ_HANDLED;
3833 }
3834
3835 /**
3836  * i40e_irq_affinity_notify - Callback for affinity changes
3837  * @notify: context as to what irq was changed
3838  * @mask: the new affinity mask
3839  *
3840  * This is a callback function used by the irq_set_affinity_notifier function
3841  * so that we may register to receive changes to the irq affinity masks.
3842  **/
3843 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3844                                      const cpumask_t *mask)
3845 {
3846         struct i40e_q_vector *q_vector =
3847                 container_of(notify, struct i40e_q_vector, affinity_notify);
3848
3849         cpumask_copy(&q_vector->affinity_mask, mask);
3850 }
3851
3852 /**
3853  * i40e_irq_affinity_release - Callback for affinity notifier release
3854  * @ref: internal core kernel usage
3855  *
3856  * This is a callback function used by the irq_set_affinity_notifier function
3857  * to inform the current notification subscriber that they will no longer
3858  * receive notifications.
3859  **/
3860 static void i40e_irq_affinity_release(struct kref *ref) {}
3861
3862 /**
3863  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3864  * @vsi: the VSI being configured
3865  * @basename: name for the vector
3866  *
3867  * Allocates MSI-X vectors and requests interrupts from the kernel.
3868  **/
3869 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3870 {
3871         int q_vectors = vsi->num_q_vectors;
3872         struct i40e_pf *pf = vsi->back;
3873         int base = vsi->base_vector;
3874         int rx_int_idx = 0;
3875         int tx_int_idx = 0;
3876         int vector, err;
3877         int irq_num;
3878         int cpu;
3879
3880         for (vector = 0; vector < q_vectors; vector++) {
3881                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3882
3883                 irq_num = pf->msix_entries[base + vector].vector;
3884
3885                 if (q_vector->tx.ring && q_vector->rx.ring) {
3886                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3887                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3888                         tx_int_idx++;
3889                 } else if (q_vector->rx.ring) {
3890                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3891                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3892                 } else if (q_vector->tx.ring) {
3893                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3894                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3895                 } else {
3896                         /* skip this unused q_vector */
3897                         continue;
3898                 }
3899                 err = request_irq(irq_num,
3900                                   vsi->irq_handler,
3901                                   0,
3902                                   q_vector->name,
3903                                   q_vector);
3904                 if (err) {
3905                         dev_info(&pf->pdev->dev,
3906                                  "MSIX request_irq failed, error: %d\n", err);
3907                         goto free_queue_irqs;
3908                 }
3909
3910                 /* register for affinity change notifications */
3911                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3912                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3913                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3914                 /* Spread affinity hints out across online CPUs.
3915                  *
3916                  * get_cpu_mask returns a static constant mask with
3917                  * a permanent lifetime so it's ok to pass to
3918                  * irq_update_affinity_hint without making a copy.
3919                  */
3920                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3921                 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
3922         }
3923
3924         vsi->irqs_ready = true;
3925         return 0;
3926
3927 free_queue_irqs:
3928         while (vector) {
3929                 vector--;
3930                 irq_num = pf->msix_entries[base + vector].vector;
3931                 irq_set_affinity_notifier(irq_num, NULL);
3932                 irq_update_affinity_hint(irq_num, NULL);
3933                 free_irq(irq_num, &vsi->q_vectors[vector]);
3934         }
3935         return err;
3936 }
3937
3938 /**
3939  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3940  * @vsi: the VSI being un-configured
3941  **/
3942 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3943 {
3944         struct i40e_pf *pf = vsi->back;
3945         struct i40e_hw *hw = &pf->hw;
3946         int base = vsi->base_vector;
3947         int i;
3948
3949         /* disable interrupt causation from each queue */
3950         for (i = 0; i < vsi->num_queue_pairs; i++) {
3951                 u32 val;
3952
3953                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3954                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3955                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3956
3957                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3958                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3959                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3960
3961                 if (!i40e_enabled_xdp_vsi(vsi))
3962                         continue;
3963                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3964         }
3965
3966         /* disable each interrupt */
3967         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3968                 for (i = vsi->base_vector;
3969                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3970                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3971
3972                 i40e_flush(hw);
3973                 for (i = 0; i < vsi->num_q_vectors; i++)
3974                         synchronize_irq(pf->msix_entries[i + base].vector);
3975         } else {
3976                 /* Legacy and MSI mode - this stops all interrupt handling */
3977                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3978                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3979                 i40e_flush(hw);
3980                 synchronize_irq(pf->pdev->irq);
3981         }
3982 }
3983
3984 /**
3985  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3986  * @vsi: the VSI being configured
3987  **/
3988 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3989 {
3990         struct i40e_pf *pf = vsi->back;
3991         int i;
3992
3993         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3994                 for (i = 0; i < vsi->num_q_vectors; i++)
3995                         i40e_irq_dynamic_enable(vsi, i);
3996         } else {
3997                 i40e_irq_dynamic_enable_icr0(pf);
3998         }
3999
4000         i40e_flush(&pf->hw);
4001         return 0;
4002 }
4003
4004 /**
4005  * i40e_free_misc_vector - Free the vector that handles non-queue events
4006  * @pf: board private structure
4007  **/
4008 static void i40e_free_misc_vector(struct i40e_pf *pf)
4009 {
4010         /* Disable ICR 0 */
4011         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4012         i40e_flush(&pf->hw);
4013
4014         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4015                 synchronize_irq(pf->msix_entries[0].vector);
4016                 free_irq(pf->msix_entries[0].vector, pf);
4017                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4018         }
4019 }
4020
4021 /**
4022  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4023  * @irq: interrupt number
4024  * @data: pointer to a q_vector
4025  *
4026  * This is the handler used for all MSI/Legacy interrupts, and deals
4027  * with both queue and non-queue interrupts.  This is also used in
4028  * MSIX mode to handle the non-queue interrupts.
4029  **/
4030 static irqreturn_t i40e_intr(int irq, void *data)
4031 {
4032         struct i40e_pf *pf = (struct i40e_pf *)data;
4033         struct i40e_hw *hw = &pf->hw;
4034         irqreturn_t ret = IRQ_NONE;
4035         u32 icr0, icr0_remaining;
4036         u32 val, ena_mask;
4037
4038         icr0 = rd32(hw, I40E_PFINT_ICR0);
4039         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4040
4041         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4042         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4043                 goto enable_intr;
4044
4045         /* if interrupt but no bits showing, must be SWINT */
4046         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4047             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4048                 pf->sw_int_count++;
4049
4050         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
4051             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4052                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4053                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4054                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4055         }
4056
4057         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4058         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4059                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4060                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4061
4062                 /* We do not have a way to disarm Queue causes while leaving
4063                  * interrupt enabled for all other causes, ideally
4064                  * interrupt should be disabled while we are in NAPI but
4065                  * this is not a performance path and napi_schedule()
4066                  * can deal with rescheduling.
4067                  */
4068                 if (!test_bit(__I40E_DOWN, pf->state))
4069                         napi_schedule_irqoff(&q_vector->napi);
4070         }
4071
4072         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4073                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4074                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4075                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4076         }
4077
4078         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4079                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4080                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4081         }
4082
4083         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4084                 /* disable any further VFLR event notifications */
4085                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4086                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4087
4088                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4089                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4090                 } else {
4091                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4092                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4093                 }
4094         }
4095
4096         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4097                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4098                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4099                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4100                 val = rd32(hw, I40E_GLGEN_RSTAT);
4101                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4102                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4103                 if (val == I40E_RESET_CORER) {
4104                         pf->corer_count++;
4105                 } else if (val == I40E_RESET_GLOBR) {
4106                         pf->globr_count++;
4107                 } else if (val == I40E_RESET_EMPR) {
4108                         pf->empr_count++;
4109                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4110                 }
4111         }
4112
4113         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4114                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4115                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4116                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4117                          rd32(hw, I40E_PFHMC_ERRORINFO),
4118                          rd32(hw, I40E_PFHMC_ERRORDATA));
4119         }
4120
4121         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4122                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4123
4124                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4125                         schedule_work(&pf->ptp_extts0_work);
4126
4127                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4128                         i40e_ptp_tx_hwtstamp(pf);
4129
4130                 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4131         }
4132
4133         /* If a critical error is pending we have no choice but to reset the
4134          * device.
4135          * Report and mask out any remaining unexpected interrupts.
4136          */
4137         icr0_remaining = icr0 & ena_mask;
4138         if (icr0_remaining) {
4139                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4140                          icr0_remaining);
4141                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4142                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4143                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4144                         dev_info(&pf->pdev->dev, "device will be reset\n");
4145                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4146                         i40e_service_event_schedule(pf);
4147                 }
4148                 ena_mask &= ~icr0_remaining;
4149         }
4150         ret = IRQ_HANDLED;
4151
4152 enable_intr:
4153         /* re-enable interrupt causes */
4154         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4155         if (!test_bit(__I40E_DOWN, pf->state) ||
4156             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4157                 i40e_service_event_schedule(pf);
4158                 i40e_irq_dynamic_enable_icr0(pf);
4159         }
4160
4161         return ret;
4162 }
4163
4164 /**
4165  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4166  * @tx_ring:  tx ring to clean
4167  * @budget:   how many cleans we're allowed
4168  *
4169  * Returns true if there's any budget left (e.g. the clean is finished)
4170  **/
4171 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4172 {
4173         struct i40e_vsi *vsi = tx_ring->vsi;
4174         u16 i = tx_ring->next_to_clean;
4175         struct i40e_tx_buffer *tx_buf;
4176         struct i40e_tx_desc *tx_desc;
4177
4178         tx_buf = &tx_ring->tx_bi[i];
4179         tx_desc = I40E_TX_DESC(tx_ring, i);
4180         i -= tx_ring->count;
4181
4182         do {
4183                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4184
4185                 /* if next_to_watch is not set then there is no work pending */
4186                 if (!eop_desc)
4187                         break;
4188
4189                 /* prevent any other reads prior to eop_desc */
4190                 smp_rmb();
4191
4192                 /* if the descriptor isn't done, no work yet to do */
4193                 if (!(eop_desc->cmd_type_offset_bsz &
4194                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4195                         break;
4196
4197                 /* clear next_to_watch to prevent false hangs */
4198                 tx_buf->next_to_watch = NULL;
4199
4200                 tx_desc->buffer_addr = 0;
4201                 tx_desc->cmd_type_offset_bsz = 0;
4202                 /* move past filter desc */
4203                 tx_buf++;
4204                 tx_desc++;
4205                 i++;
4206                 if (unlikely(!i)) {
4207                         i -= tx_ring->count;
4208                         tx_buf = tx_ring->tx_bi;
4209                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4210                 }
4211                 /* unmap skb header data */
4212                 dma_unmap_single(tx_ring->dev,
4213                                  dma_unmap_addr(tx_buf, dma),
4214                                  dma_unmap_len(tx_buf, len),
4215                                  DMA_TO_DEVICE);
4216                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4217                         kfree(tx_buf->raw_buf);
4218
4219                 tx_buf->raw_buf = NULL;
4220                 tx_buf->tx_flags = 0;
4221                 tx_buf->next_to_watch = NULL;
4222                 dma_unmap_len_set(tx_buf, len, 0);
4223                 tx_desc->buffer_addr = 0;
4224                 tx_desc->cmd_type_offset_bsz = 0;
4225
4226                 /* move us past the eop_desc for start of next FD desc */
4227                 tx_buf++;
4228                 tx_desc++;
4229                 i++;
4230                 if (unlikely(!i)) {
4231                         i -= tx_ring->count;
4232                         tx_buf = tx_ring->tx_bi;
4233                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4234                 }
4235
4236                 /* update budget accounting */
4237                 budget--;
4238         } while (likely(budget));
4239
4240         i += tx_ring->count;
4241         tx_ring->next_to_clean = i;
4242
4243         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4244                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4245
4246         return budget > 0;
4247 }
4248
4249 /**
4250  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4251  * @irq: interrupt number
4252  * @data: pointer to a q_vector
4253  **/
4254 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4255 {
4256         struct i40e_q_vector *q_vector = data;
4257         struct i40e_vsi *vsi;
4258
4259         if (!q_vector->tx.ring)
4260                 return IRQ_HANDLED;
4261
4262         vsi = q_vector->tx.ring->vsi;
4263         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4264
4265         return IRQ_HANDLED;
4266 }
4267
4268 /**
4269  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4270  * @vsi: the VSI being configured
4271  * @v_idx: vector index
4272  * @qp_idx: queue pair index
4273  **/
4274 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4275 {
4276         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4277         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4278         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4279
4280         tx_ring->q_vector = q_vector;
4281         tx_ring->next = q_vector->tx.ring;
4282         q_vector->tx.ring = tx_ring;
4283         q_vector->tx.count++;
4284
4285         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4286         if (i40e_enabled_xdp_vsi(vsi)) {
4287                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4288
4289                 xdp_ring->q_vector = q_vector;
4290                 xdp_ring->next = q_vector->tx.ring;
4291                 q_vector->tx.ring = xdp_ring;
4292                 q_vector->tx.count++;
4293         }
4294
4295         rx_ring->q_vector = q_vector;
4296         rx_ring->next = q_vector->rx.ring;
4297         q_vector->rx.ring = rx_ring;
4298         q_vector->rx.count++;
4299 }
4300
4301 /**
4302  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4303  * @vsi: the VSI being configured
4304  *
4305  * This function maps descriptor rings to the queue-specific vectors
4306  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4307  * one vector per queue pair, but on a constrained vector budget, we
4308  * group the queue pairs as "efficiently" as possible.
4309  **/
4310 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4311 {
4312         int qp_remaining = vsi->num_queue_pairs;
4313         int q_vectors = vsi->num_q_vectors;
4314         int num_ringpairs;
4315         int v_start = 0;
4316         int qp_idx = 0;
4317
4318         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4319          * group them so there are multiple queues per vector.
4320          * It is also important to go through all the vectors available to be
4321          * sure that if we don't use all the vectors, that the remaining vectors
4322          * are cleared. This is especially important when decreasing the
4323          * number of queues in use.
4324          */
4325         for (; v_start < q_vectors; v_start++) {
4326                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4327
4328                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4329
4330                 q_vector->num_ringpairs = num_ringpairs;
4331                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4332
4333                 q_vector->rx.count = 0;
4334                 q_vector->tx.count = 0;
4335                 q_vector->rx.ring = NULL;
4336                 q_vector->tx.ring = NULL;
4337
4338                 while (num_ringpairs--) {
4339                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4340                         qp_idx++;
4341                         qp_remaining--;
4342                 }
4343         }
4344 }
4345
4346 /**
4347  * i40e_vsi_request_irq - Request IRQ from the OS
4348  * @vsi: the VSI being configured
4349  * @basename: name for the vector
4350  **/
4351 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4352 {
4353         struct i40e_pf *pf = vsi->back;
4354         int err;
4355
4356         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4357                 err = i40e_vsi_request_irq_msix(vsi, basename);
4358         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4359                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4360                                   pf->int_name, pf);
4361         else
4362                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4363                                   pf->int_name, pf);
4364
4365         if (err)
4366                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4367
4368         return err;
4369 }
4370
4371 #ifdef CONFIG_NET_POLL_CONTROLLER
4372 /**
4373  * i40e_netpoll - A Polling 'interrupt' handler
4374  * @netdev: network interface device structure
4375  *
4376  * This is used by netconsole to send skbs without having to re-enable
4377  * interrupts.  It's not called while the normal interrupt routine is executing.
4378  **/
4379 static void i40e_netpoll(struct net_device *netdev)
4380 {
4381         struct i40e_netdev_priv *np = netdev_priv(netdev);
4382         struct i40e_vsi *vsi = np->vsi;
4383         struct i40e_pf *pf = vsi->back;
4384         int i;
4385
4386         /* if interface is down do nothing */
4387         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4388                 return;
4389
4390         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4391                 for (i = 0; i < vsi->num_q_vectors; i++)
4392                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4393         } else {
4394                 i40e_intr(pf->pdev->irq, netdev);
4395         }
4396 }
4397 #endif
4398
4399 #define I40E_QTX_ENA_WAIT_COUNT 50
4400
4401 /**
4402  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4403  * @pf: the PF being configured
4404  * @pf_q: the PF queue
4405  * @enable: enable or disable state of the queue
4406  *
4407  * This routine will wait for the given Tx queue of the PF to reach the
4408  * enabled or disabled state.
4409  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4410  * multiple retries; else will return 0 in case of success.
4411  **/
4412 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4413 {
4414         int i;
4415         u32 tx_reg;
4416
4417         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4418                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4419                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4420                         break;
4421
4422                 usleep_range(10, 20);
4423         }
4424         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4425                 return -ETIMEDOUT;
4426
4427         return 0;
4428 }
4429
4430 /**
4431  * i40e_control_tx_q - Start or stop a particular Tx queue
4432  * @pf: the PF structure
4433  * @pf_q: the PF queue to configure
4434  * @enable: start or stop the queue
4435  *
4436  * This function enables or disables a single queue. Note that any delay
4437  * required after the operation is expected to be handled by the caller of
4438  * this function.
4439  **/
4440 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4441 {
4442         struct i40e_hw *hw = &pf->hw;
4443         u32 tx_reg;
4444         int i;
4445
4446         /* warn the TX unit of coming changes */
4447         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4448         if (!enable)
4449                 usleep_range(10, 20);
4450
4451         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4452                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4453                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4454                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4455                         break;
4456                 usleep_range(1000, 2000);
4457         }
4458
4459         /* Skip if the queue is already in the requested state */
4460         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4461                 return;
4462
4463         /* turn on/off the queue */
4464         if (enable) {
4465                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4466                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4467         } else {
4468                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4469         }
4470
4471         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4472 }
4473
4474 /**
4475  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4476  * @seid: VSI SEID
4477  * @pf: the PF structure
4478  * @pf_q: the PF queue to configure
4479  * @is_xdp: true if the queue is used for XDP
4480  * @enable: start or stop the queue
4481  **/
4482 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4483                            bool is_xdp, bool enable)
4484 {
4485         int ret;
4486
4487         i40e_control_tx_q(pf, pf_q, enable);
4488
4489         /* wait for the change to finish */
4490         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4491         if (ret) {
4492                 dev_info(&pf->pdev->dev,
4493                          "VSI seid %d %sTx ring %d %sable timeout\n",
4494                          seid, (is_xdp ? "XDP " : ""), pf_q,
4495                          (enable ? "en" : "dis"));
4496         }
4497
4498         return ret;
4499 }
4500
4501 /**
4502  * i40e_vsi_enable_tx - Start a VSI's rings
4503  * @vsi: the VSI being configured
4504  **/
4505 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4506 {
4507         struct i40e_pf *pf = vsi->back;
4508         int i, pf_q, ret = 0;
4509
4510         pf_q = vsi->base_queue;
4511         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4512                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4513                                              pf_q,
4514                                              false /*is xdp*/, true);
4515                 if (ret)
4516                         break;
4517
4518                 if (!i40e_enabled_xdp_vsi(vsi))
4519                         continue;
4520
4521                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4522                                              pf_q + vsi->alloc_queue_pairs,
4523                                              true /*is xdp*/, true);
4524                 if (ret)
4525                         break;
4526         }
4527         return ret;
4528 }
4529
4530 /**
4531  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4532  * @pf: the PF being configured
4533  * @pf_q: the PF queue
4534  * @enable: enable or disable state of the queue
4535  *
4536  * This routine will wait for the given Rx queue of the PF to reach the
4537  * enabled or disabled state.
4538  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4539  * multiple retries; else will return 0 in case of success.
4540  **/
4541 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4542 {
4543         int i;
4544         u32 rx_reg;
4545
4546         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4547                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4548                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4549                         break;
4550
4551                 usleep_range(10, 20);
4552         }
4553         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4554                 return -ETIMEDOUT;
4555
4556         return 0;
4557 }
4558
4559 /**
4560  * i40e_control_rx_q - Start or stop a particular Rx queue
4561  * @pf: the PF structure
4562  * @pf_q: the PF queue to configure
4563  * @enable: start or stop the queue
4564  *
4565  * This function enables or disables a single queue. Note that
4566  * any delay required after the operation is expected to be
4567  * handled by the caller of this function.
4568  **/
4569 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4570 {
4571         struct i40e_hw *hw = &pf->hw;
4572         u32 rx_reg;
4573         int i;
4574
4575         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4576                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4577                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4578                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4579                         break;
4580                 usleep_range(1000, 2000);
4581         }
4582
4583         /* Skip if the queue is already in the requested state */
4584         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4585                 return;
4586
4587         /* turn on/off the queue */
4588         if (enable)
4589                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4590         else
4591                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4592
4593         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4594 }
4595
4596 /**
4597  * i40e_control_wait_rx_q
4598  * @pf: the PF structure
4599  * @pf_q: queue being configured
4600  * @enable: start or stop the rings
4601  *
4602  * This function enables or disables a single queue along with waiting
4603  * for the change to finish. The caller of this function should handle
4604  * the delays needed in the case of disabling queues.
4605  **/
4606 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4607 {
4608         int ret = 0;
4609
4610         i40e_control_rx_q(pf, pf_q, enable);
4611
4612         /* wait for the change to finish */
4613         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4614         if (ret)
4615                 return ret;
4616
4617         return ret;
4618 }
4619
4620 /**
4621  * i40e_vsi_enable_rx - Start a VSI's rings
4622  * @vsi: the VSI being configured
4623  **/
4624 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4625 {
4626         struct i40e_pf *pf = vsi->back;
4627         int i, pf_q, ret = 0;
4628
4629         pf_q = vsi->base_queue;
4630         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4631                 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4632                 if (ret) {
4633                         dev_info(&pf->pdev->dev,
4634                                  "VSI seid %d Rx ring %d enable timeout\n",
4635                                  vsi->seid, pf_q);
4636                         break;
4637                 }
4638         }
4639
4640         return ret;
4641 }
4642
4643 /**
4644  * i40e_vsi_start_rings - Start a VSI's rings
4645  * @vsi: the VSI being configured
4646  **/
4647 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4648 {
4649         int ret = 0;
4650
4651         /* do rx first for enable and last for disable */
4652         ret = i40e_vsi_enable_rx(vsi);
4653         if (ret)
4654                 return ret;
4655         ret = i40e_vsi_enable_tx(vsi);
4656
4657         return ret;
4658 }
4659
4660 #define I40E_DISABLE_TX_GAP_MSEC        50
4661
4662 /**
4663  * i40e_vsi_stop_rings - Stop a VSI's rings
4664  * @vsi: the VSI being configured
4665  **/
4666 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4667 {
4668         struct i40e_pf *pf = vsi->back;
4669         int pf_q, err, q_end;
4670
4671         /* When port TX is suspended, don't wait */
4672         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4673                 return i40e_vsi_stop_rings_no_wait(vsi);
4674
4675         q_end = vsi->base_queue + vsi->num_queue_pairs;
4676         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4677                 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4678
4679         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4680                 err = i40e_control_wait_rx_q(pf, pf_q, false);
4681                 if (err)
4682                         dev_info(&pf->pdev->dev,
4683                                  "VSI seid %d Rx ring %d disable timeout\n",
4684                                  vsi->seid, pf_q);
4685         }
4686
4687         msleep(I40E_DISABLE_TX_GAP_MSEC);
4688         pf_q = vsi->base_queue;
4689         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4690                 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4691
4692         i40e_vsi_wait_queues_disabled(vsi);
4693 }
4694
4695 /**
4696  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4697  * @vsi: the VSI being shutdown
4698  *
4699  * This function stops all the rings for a VSI but does not delay to verify
4700  * that rings have been disabled. It is expected that the caller is shutting
4701  * down multiple VSIs at once and will delay together for all the VSIs after
4702  * initiating the shutdown. This is particularly useful for shutting down lots
4703  * of VFs together. Otherwise, a large delay can be incurred while configuring
4704  * each VSI in serial.
4705  **/
4706 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4707 {
4708         struct i40e_pf *pf = vsi->back;
4709         int i, pf_q;
4710
4711         pf_q = vsi->base_queue;
4712         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4713                 i40e_control_tx_q(pf, pf_q, false);
4714                 i40e_control_rx_q(pf, pf_q, false);
4715         }
4716 }
4717
4718 /**
4719  * i40e_vsi_free_irq - Free the irq association with the OS
4720  * @vsi: the VSI being configured
4721  **/
4722 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4723 {
4724         struct i40e_pf *pf = vsi->back;
4725         struct i40e_hw *hw = &pf->hw;
4726         int base = vsi->base_vector;
4727         u32 val, qp;
4728         int i;
4729
4730         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4731                 if (!vsi->q_vectors)
4732                         return;
4733
4734                 if (!vsi->irqs_ready)
4735                         return;
4736
4737                 vsi->irqs_ready = false;
4738                 for (i = 0; i < vsi->num_q_vectors; i++) {
4739                         int irq_num;
4740                         u16 vector;
4741
4742                         vector = i + base;
4743                         irq_num = pf->msix_entries[vector].vector;
4744
4745                         /* free only the irqs that were actually requested */
4746                         if (!vsi->q_vectors[i] ||
4747                             !vsi->q_vectors[i]->num_ringpairs)
4748                                 continue;
4749
4750                         /* clear the affinity notifier in the IRQ descriptor */
4751                         irq_set_affinity_notifier(irq_num, NULL);
4752                         /* remove our suggested affinity mask for this IRQ */
4753                         irq_update_affinity_hint(irq_num, NULL);
4754                         synchronize_irq(irq_num);
4755                         free_irq(irq_num, vsi->q_vectors[i]);
4756
4757                         /* Tear down the interrupt queue link list
4758                          *
4759                          * We know that they come in pairs and always
4760                          * the Rx first, then the Tx.  To clear the
4761                          * link list, stick the EOL value into the
4762                          * next_q field of the registers.
4763                          */
4764                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4765                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4766                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4767                         val |= I40E_QUEUE_END_OF_LIST
4768                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4769                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4770
4771                         while (qp != I40E_QUEUE_END_OF_LIST) {
4772                                 u32 next;
4773
4774                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4775
4776                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4777                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4778                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4779                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4780
4781                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4782                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4783
4784                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4785
4786                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4787
4788                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4789                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4790
4791                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4792                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4793                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4794                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4795
4796                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4797                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4798
4799                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4800                                 qp = next;
4801                         }
4802                 }
4803         } else {
4804                 free_irq(pf->pdev->irq, pf);
4805
4806                 val = rd32(hw, I40E_PFINT_LNKLST0);
4807                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4808                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4809                 val |= I40E_QUEUE_END_OF_LIST
4810                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4811                 wr32(hw, I40E_PFINT_LNKLST0, val);
4812
4813                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4814                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4815                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4816                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4817                          I40E_QINT_RQCTL_INTEVENT_MASK);
4818
4819                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4820                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4821
4822                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4823
4824                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4825
4826                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4827                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4828                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4829                          I40E_QINT_TQCTL_INTEVENT_MASK);
4830
4831                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4832                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4833
4834                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4835         }
4836 }
4837
4838 /**
4839  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4840  * @vsi: the VSI being configured
4841  * @v_idx: Index of vector to be freed
4842  *
4843  * This function frees the memory allocated to the q_vector.  In addition if
4844  * NAPI is enabled it will delete any references to the NAPI struct prior
4845  * to freeing the q_vector.
4846  **/
4847 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4848 {
4849         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4850         struct i40e_ring *ring;
4851
4852         if (!q_vector)
4853                 return;
4854
4855         /* disassociate q_vector from rings */
4856         i40e_for_each_ring(ring, q_vector->tx)
4857                 ring->q_vector = NULL;
4858
4859         i40e_for_each_ring(ring, q_vector->rx)
4860                 ring->q_vector = NULL;
4861
4862         /* only VSI w/ an associated netdev is set up w/ NAPI */
4863         if (vsi->netdev)
4864                 netif_napi_del(&q_vector->napi);
4865
4866         vsi->q_vectors[v_idx] = NULL;
4867
4868         kfree_rcu(q_vector, rcu);
4869 }
4870
4871 /**
4872  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4873  * @vsi: the VSI being un-configured
4874  *
4875  * This frees the memory allocated to the q_vectors and
4876  * deletes references to the NAPI struct.
4877  **/
4878 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4879 {
4880         int v_idx;
4881
4882         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4883                 i40e_free_q_vector(vsi, v_idx);
4884 }
4885
4886 /**
4887  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4888  * @pf: board private structure
4889  **/
4890 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4891 {
4892         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4893         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4894                 pci_disable_msix(pf->pdev);
4895                 kfree(pf->msix_entries);
4896                 pf->msix_entries = NULL;
4897                 kfree(pf->irq_pile);
4898                 pf->irq_pile = NULL;
4899         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4900                 pci_disable_msi(pf->pdev);
4901         }
4902         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4903 }
4904
4905 /**
4906  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4907  * @pf: board private structure
4908  *
4909  * We go through and clear interrupt specific resources and reset the structure
4910  * to pre-load conditions
4911  **/
4912 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4913 {
4914         int i;
4915
4916         if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
4917                 i40e_free_misc_vector(pf);
4918
4919         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4920                       I40E_IWARP_IRQ_PILE_ID);
4921
4922         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4923         for (i = 0; i < pf->num_alloc_vsi; i++)
4924                 if (pf->vsi[i])
4925                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4926         i40e_reset_interrupt_capability(pf);
4927 }
4928
4929 /**
4930  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4931  * @vsi: the VSI being configured
4932  **/
4933 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4934 {
4935         int q_idx;
4936
4937         if (!vsi->netdev)
4938                 return;
4939
4940         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4941                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4942
4943                 if (q_vector->rx.ring || q_vector->tx.ring)
4944                         napi_enable(&q_vector->napi);
4945         }
4946 }
4947
4948 /**
4949  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4950  * @vsi: the VSI being configured
4951  **/
4952 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4953 {
4954         int q_idx;
4955
4956         if (!vsi->netdev)
4957                 return;
4958
4959         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4960                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4961
4962                 if (q_vector->rx.ring || q_vector->tx.ring)
4963                         napi_disable(&q_vector->napi);
4964         }
4965 }
4966
4967 /**
4968  * i40e_vsi_close - Shut down a VSI
4969  * @vsi: the vsi to be quelled
4970  **/
4971 static void i40e_vsi_close(struct i40e_vsi *vsi)
4972 {
4973         struct i40e_pf *pf = vsi->back;
4974         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4975                 i40e_down(vsi);
4976         i40e_vsi_free_irq(vsi);
4977         i40e_vsi_free_tx_resources(vsi);
4978         i40e_vsi_free_rx_resources(vsi);
4979         vsi->current_netdev_flags = 0;
4980         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4981         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4982                 set_bit(__I40E_CLIENT_RESET, pf->state);
4983 }
4984
4985 /**
4986  * i40e_quiesce_vsi - Pause a given VSI
4987  * @vsi: the VSI being paused
4988  **/
4989 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4990 {
4991         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4992                 return;
4993
4994         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4995         if (vsi->netdev && netif_running(vsi->netdev))
4996                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4997         else
4998                 i40e_vsi_close(vsi);
4999 }
5000
5001 /**
5002  * i40e_unquiesce_vsi - Resume a given VSI
5003  * @vsi: the VSI being resumed
5004  **/
5005 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5006 {
5007         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5008                 return;
5009
5010         if (vsi->netdev && netif_running(vsi->netdev))
5011                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5012         else
5013                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
5014 }
5015
5016 /**
5017  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5018  * @pf: the PF
5019  **/
5020 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5021 {
5022         int v;
5023
5024         for (v = 0; v < pf->num_alloc_vsi; v++) {
5025                 if (pf->vsi[v])
5026                         i40e_quiesce_vsi(pf->vsi[v]);
5027         }
5028 }
5029
5030 /**
5031  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5032  * @pf: the PF
5033  **/
5034 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5035 {
5036         int v;
5037
5038         for (v = 0; v < pf->num_alloc_vsi; v++) {
5039                 if (pf->vsi[v])
5040                         i40e_unquiesce_vsi(pf->vsi[v]);
5041         }
5042 }
5043
5044 /**
5045  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5046  * @vsi: the VSI being configured
5047  *
5048  * Wait until all queues on a given VSI have been disabled.
5049  **/
5050 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5051 {
5052         struct i40e_pf *pf = vsi->back;
5053         int i, pf_q, ret;
5054
5055         pf_q = vsi->base_queue;
5056         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5057                 /* Check and wait for the Tx queue */
5058                 ret = i40e_pf_txq_wait(pf, pf_q, false);
5059                 if (ret) {
5060                         dev_info(&pf->pdev->dev,
5061                                  "VSI seid %d Tx ring %d disable timeout\n",
5062                                  vsi->seid, pf_q);
5063                         return ret;
5064                 }
5065
5066                 if (!i40e_enabled_xdp_vsi(vsi))
5067                         goto wait_rx;
5068
5069                 /* Check and wait for the XDP Tx queue */
5070                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5071                                        false);
5072                 if (ret) {
5073                         dev_info(&pf->pdev->dev,
5074                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
5075                                  vsi->seid, pf_q);
5076                         return ret;
5077                 }
5078 wait_rx:
5079                 /* Check and wait for the Rx queue */
5080                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5081                 if (ret) {
5082                         dev_info(&pf->pdev->dev,
5083                                  "VSI seid %d Rx ring %d disable timeout\n",
5084                                  vsi->seid, pf_q);
5085                         return ret;
5086                 }
5087         }
5088
5089         return 0;
5090 }
5091
5092 #ifdef CONFIG_I40E_DCB
5093 /**
5094  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5095  * @pf: the PF
5096  *
5097  * This function waits for the queues to be in disabled state for all the
5098  * VSIs that are managed by this PF.
5099  **/
5100 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5101 {
5102         int v, ret = 0;
5103
5104         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5105                 if (pf->vsi[v]) {
5106                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5107                         if (ret)
5108                                 break;
5109                 }
5110         }
5111
5112         return ret;
5113 }
5114
5115 #endif
5116
5117 /**
5118  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5119  * @pf: pointer to PF
5120  *
5121  * Get TC map for ISCSI PF type that will include iSCSI TC
5122  * and LAN TC.
5123  **/
5124 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5125 {
5126         struct i40e_dcb_app_priority_table app;
5127         struct i40e_hw *hw = &pf->hw;
5128         u8 enabled_tc = 1; /* TC0 is always enabled */
5129         u8 tc, i;
5130         /* Get the iSCSI APP TLV */
5131         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5132
5133         for (i = 0; i < dcbcfg->numapps; i++) {
5134                 app = dcbcfg->app[i];
5135                 if (app.selector == I40E_APP_SEL_TCPIP &&
5136                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5137                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5138                         enabled_tc |= BIT(tc);
5139                         break;
5140                 }
5141         }
5142
5143         return enabled_tc;
5144 }
5145
5146 /**
5147  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5148  * @dcbcfg: the corresponding DCBx configuration structure
5149  *
5150  * Return the number of TCs from given DCBx configuration
5151  **/
5152 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5153 {
5154         int i, tc_unused = 0;
5155         u8 num_tc = 0;
5156         u8 ret = 0;
5157
5158         /* Scan the ETS Config Priority Table to find
5159          * traffic class enabled for a given priority
5160          * and create a bitmask of enabled TCs
5161          */
5162         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5163                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5164
5165         /* Now scan the bitmask to check for
5166          * contiguous TCs starting with TC0
5167          */
5168         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5169                 if (num_tc & BIT(i)) {
5170                         if (!tc_unused) {
5171                                 ret++;
5172                         } else {
5173                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5174                                 return 1;
5175                         }
5176                 } else {
5177                         tc_unused = 1;
5178                 }
5179         }
5180
5181         /* There is always at least TC0 */
5182         if (!ret)
5183                 ret = 1;
5184
5185         return ret;
5186 }
5187
5188 /**
5189  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5190  * @dcbcfg: the corresponding DCBx configuration structure
5191  *
5192  * Query the current DCB configuration and return the number of
5193  * traffic classes enabled from the given DCBX config
5194  **/
5195 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5196 {
5197         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5198         u8 enabled_tc = 1;
5199         u8 i;
5200
5201         for (i = 0; i < num_tc; i++)
5202                 enabled_tc |= BIT(i);
5203
5204         return enabled_tc;
5205 }
5206
5207 /**
5208  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5209  * @pf: PF being queried
5210  *
5211  * Query the current MQPRIO configuration and return the number of
5212  * traffic classes enabled.
5213  **/
5214 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5215 {
5216         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5217         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5218         u8 enabled_tc = 1, i;
5219
5220         for (i = 1; i < num_tc; i++)
5221                 enabled_tc |= BIT(i);
5222         return enabled_tc;
5223 }
5224
5225 /**
5226  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5227  * @pf: PF being queried
5228  *
5229  * Return number of traffic classes enabled for the given PF
5230  **/
5231 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5232 {
5233         struct i40e_hw *hw = &pf->hw;
5234         u8 i, enabled_tc = 1;
5235         u8 num_tc = 0;
5236         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5237
5238         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5239                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5240
5241         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5242         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5243                 return 1;
5244
5245         /* SFP mode will be enabled for all TCs on port */
5246         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5247                 return i40e_dcb_get_num_tc(dcbcfg);
5248
5249         /* MFP mode return count of enabled TCs for this PF */
5250         if (pf->hw.func_caps.iscsi)
5251                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5252         else
5253                 return 1; /* Only TC0 */
5254
5255         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5256                 if (enabled_tc & BIT(i))
5257                         num_tc++;
5258         }
5259         return num_tc;
5260 }
5261
5262 /**
5263  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5264  * @pf: PF being queried
5265  *
5266  * Return a bitmap for enabled traffic classes for this PF.
5267  **/
5268 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5269 {
5270         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5271                 return i40e_mqprio_get_enabled_tc(pf);
5272
5273         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5274          * default TC
5275          */
5276         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5277                 return I40E_DEFAULT_TRAFFIC_CLASS;
5278
5279         /* SFP mode we want PF to be enabled for all TCs */
5280         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5281                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5282
5283         /* MFP enabled and iSCSI PF type */
5284         if (pf->hw.func_caps.iscsi)
5285                 return i40e_get_iscsi_tc_map(pf);
5286         else
5287                 return I40E_DEFAULT_TRAFFIC_CLASS;
5288 }
5289
5290 /**
5291  * i40e_vsi_get_bw_info - Query VSI BW Information
5292  * @vsi: the VSI being queried
5293  *
5294  * Returns 0 on success, negative value on failure
5295  **/
5296 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5297 {
5298         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5299         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5300         struct i40e_pf *pf = vsi->back;
5301         struct i40e_hw *hw = &pf->hw;
5302         i40e_status ret;
5303         u32 tc_bw_max;
5304         int i;
5305
5306         /* Get the VSI level BW configuration */
5307         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5308         if (ret) {
5309                 dev_info(&pf->pdev->dev,
5310                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5311                          i40e_stat_str(&pf->hw, ret),
5312                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5313                 return -EINVAL;
5314         }
5315
5316         /* Get the VSI level BW configuration per TC */
5317         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5318                                                NULL);
5319         if (ret) {
5320                 dev_info(&pf->pdev->dev,
5321                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5322                          i40e_stat_str(&pf->hw, ret),
5323                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5324                 return -EINVAL;
5325         }
5326
5327         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5328                 dev_info(&pf->pdev->dev,
5329                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5330                          bw_config.tc_valid_bits,
5331                          bw_ets_config.tc_valid_bits);
5332                 /* Still continuing */
5333         }
5334
5335         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5336         vsi->bw_max_quanta = bw_config.max_bw;
5337         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5338                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5339         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5340                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5341                 vsi->bw_ets_limit_credits[i] =
5342                                         le16_to_cpu(bw_ets_config.credits[i]);
5343                 /* 3 bits out of 4 for each TC */
5344                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5345         }
5346
5347         return 0;
5348 }
5349
5350 /**
5351  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5352  * @vsi: the VSI being configured
5353  * @enabled_tc: TC bitmap
5354  * @bw_share: BW shared credits per TC
5355  *
5356  * Returns 0 on success, negative value on failure
5357  **/
5358 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5359                                        u8 *bw_share)
5360 {
5361         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5362         struct i40e_pf *pf = vsi->back;
5363         i40e_status ret;
5364         int i;
5365
5366         /* There is no need to reset BW when mqprio mode is on.  */
5367         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5368                 return 0;
5369         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5370                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5371                 if (ret)
5372                         dev_info(&pf->pdev->dev,
5373                                  "Failed to reset tx rate for vsi->seid %u\n",
5374                                  vsi->seid);
5375                 return ret;
5376         }
5377         memset(&bw_data, 0, sizeof(bw_data));
5378         bw_data.tc_valid_bits = enabled_tc;
5379         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5380                 bw_data.tc_bw_credits[i] = bw_share[i];
5381
5382         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5383         if (ret) {
5384                 dev_info(&pf->pdev->dev,
5385                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5386                          pf->hw.aq.asq_last_status);
5387                 return -EINVAL;
5388         }
5389
5390         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5391                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5392
5393         return 0;
5394 }
5395
5396 /**
5397  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5398  * @vsi: the VSI being configured
5399  * @enabled_tc: TC map to be enabled
5400  *
5401  **/
5402 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5403 {
5404         struct net_device *netdev = vsi->netdev;
5405         struct i40e_pf *pf = vsi->back;
5406         struct i40e_hw *hw = &pf->hw;
5407         u8 netdev_tc = 0;
5408         int i;
5409         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5410
5411         if (!netdev)
5412                 return;
5413
5414         if (!enabled_tc) {
5415                 netdev_reset_tc(netdev);
5416                 return;
5417         }
5418
5419         /* Set up actual enabled TCs on the VSI */
5420         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5421                 return;
5422
5423         /* set per TC queues for the VSI */
5424         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5425                 /* Only set TC queues for enabled tcs
5426                  *
5427                  * e.g. For a VSI that has TC0 and TC3 enabled the
5428                  * enabled_tc bitmap would be 0x00001001; the driver
5429                  * will set the numtc for netdev as 2 that will be
5430                  * referenced by the netdev layer as TC 0 and 1.
5431                  */
5432                 if (vsi->tc_config.enabled_tc & BIT(i))
5433                         netdev_set_tc_queue(netdev,
5434                                         vsi->tc_config.tc_info[i].netdev_tc,
5435                                         vsi->tc_config.tc_info[i].qcount,
5436                                         vsi->tc_config.tc_info[i].qoffset);
5437         }
5438
5439         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5440                 return;
5441
5442         /* Assign UP2TC map for the VSI */
5443         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5444                 /* Get the actual TC# for the UP */
5445                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5446                 /* Get the mapped netdev TC# for the UP */
5447                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5448                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5449         }
5450 }
5451
5452 /**
5453  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5454  * @vsi: the VSI being configured
5455  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5456  **/
5457 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5458                                       struct i40e_vsi_context *ctxt)
5459 {
5460         /* copy just the sections touched not the entire info
5461          * since not all sections are valid as returned by
5462          * update vsi params
5463          */
5464         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5465         memcpy(&vsi->info.queue_mapping,
5466                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5467         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5468                sizeof(vsi->info.tc_mapping));
5469 }
5470
5471 /**
5472  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5473  * @vsi: the VSI being reconfigured
5474  * @vsi_offset: offset from main VF VSI
5475  */
5476 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5477 {
5478         struct i40e_vsi_context ctxt = {};
5479         struct i40e_pf *pf;
5480         struct i40e_hw *hw;
5481         int ret;
5482
5483         if (!vsi)
5484                 return I40E_ERR_PARAM;
5485         pf = vsi->back;
5486         hw = &pf->hw;
5487
5488         ctxt.seid = vsi->seid;
5489         ctxt.pf_num = hw->pf_id;
5490         ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5491         ctxt.uplink_seid = vsi->uplink_seid;
5492         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5493         ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5494         ctxt.info = vsi->info;
5495
5496         i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5497                                  false);
5498         if (vsi->reconfig_rss) {
5499                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5500                                       vsi->num_queue_pairs);
5501                 ret = i40e_vsi_config_rss(vsi);
5502                 if (ret) {
5503                         dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5504                         return ret;
5505                 }
5506                 vsi->reconfig_rss = false;
5507         }
5508
5509         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5510         if (ret) {
5511                 dev_info(&pf->pdev->dev, "Update vsi config failed, err %s aq_err %s\n",
5512                          i40e_stat_str(hw, ret),
5513                          i40e_aq_str(hw, hw->aq.asq_last_status));
5514                 return ret;
5515         }
5516         /* update the local VSI info with updated queue map */
5517         i40e_vsi_update_queue_map(vsi, &ctxt);
5518         vsi->info.valid_sections = 0;
5519
5520         return ret;
5521 }
5522
5523 /**
5524  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5525  * @vsi: VSI to be configured
5526  * @enabled_tc: TC bitmap
5527  *
5528  * This configures a particular VSI for TCs that are mapped to the
5529  * given TC bitmap. It uses default bandwidth share for TCs across
5530  * VSIs to configure TC for a particular VSI.
5531  *
5532  * NOTE:
5533  * It is expected that the VSI queues have been quisced before calling
5534  * this function.
5535  **/
5536 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5537 {
5538         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5539         struct i40e_pf *pf = vsi->back;
5540         struct i40e_hw *hw = &pf->hw;
5541         struct i40e_vsi_context ctxt;
5542         int ret = 0;
5543         int i;
5544
5545         /* Check if enabled_tc is same as existing or new TCs */
5546         if (vsi->tc_config.enabled_tc == enabled_tc &&
5547             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5548                 return ret;
5549
5550         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5551         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5552                 if (enabled_tc & BIT(i))
5553                         bw_share[i] = 1;
5554         }
5555
5556         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5557         if (ret) {
5558                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5559
5560                 dev_info(&pf->pdev->dev,
5561                          "Failed configuring TC map %d for VSI %d\n",
5562                          enabled_tc, vsi->seid);
5563                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5564                                                   &bw_config, NULL);
5565                 if (ret) {
5566                         dev_info(&pf->pdev->dev,
5567                                  "Failed querying vsi bw info, err %s aq_err %s\n",
5568                                  i40e_stat_str(hw, ret),
5569                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5570                         goto out;
5571                 }
5572                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5573                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5574
5575                         if (!valid_tc)
5576                                 valid_tc = bw_config.tc_valid_bits;
5577                         /* Always enable TC0, no matter what */
5578                         valid_tc |= 1;
5579                         dev_info(&pf->pdev->dev,
5580                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5581                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5582                         enabled_tc = valid_tc;
5583                 }
5584
5585                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5586                 if (ret) {
5587                         dev_err(&pf->pdev->dev,
5588                                 "Unable to  configure TC map %d for VSI %d\n",
5589                                 enabled_tc, vsi->seid);
5590                         goto out;
5591                 }
5592         }
5593
5594         /* Update Queue Pairs Mapping for currently enabled UPs */
5595         ctxt.seid = vsi->seid;
5596         ctxt.pf_num = vsi->back->hw.pf_id;
5597         ctxt.vf_num = 0;
5598         ctxt.uplink_seid = vsi->uplink_seid;
5599         ctxt.info = vsi->info;
5600         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5601                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5602                 if (ret)
5603                         goto out;
5604         } else {
5605                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5606         }
5607
5608         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5609          * queues changed.
5610          */
5611         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5612                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5613                                       vsi->num_queue_pairs);
5614                 ret = i40e_vsi_config_rss(vsi);
5615                 if (ret) {
5616                         dev_info(&vsi->back->pdev->dev,
5617                                  "Failed to reconfig rss for num_queues\n");
5618                         return ret;
5619                 }
5620                 vsi->reconfig_rss = false;
5621         }
5622         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5623                 ctxt.info.valid_sections |=
5624                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5625                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5626         }
5627
5628         /* Update the VSI after updating the VSI queue-mapping
5629          * information
5630          */
5631         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5632         if (ret) {
5633                 dev_info(&pf->pdev->dev,
5634                          "Update vsi tc config failed, err %s aq_err %s\n",
5635                          i40e_stat_str(hw, ret),
5636                          i40e_aq_str(hw, hw->aq.asq_last_status));
5637                 goto out;
5638         }
5639         /* update the local VSI info with updated queue map */
5640         i40e_vsi_update_queue_map(vsi, &ctxt);
5641         vsi->info.valid_sections = 0;
5642
5643         /* Update current VSI BW information */
5644         ret = i40e_vsi_get_bw_info(vsi);
5645         if (ret) {
5646                 dev_info(&pf->pdev->dev,
5647                          "Failed updating vsi bw info, err %s aq_err %s\n",
5648                          i40e_stat_str(hw, ret),
5649                          i40e_aq_str(hw, hw->aq.asq_last_status));
5650                 goto out;
5651         }
5652
5653         /* Update the netdev TC setup */
5654         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5655 out:
5656         return ret;
5657 }
5658
5659 /**
5660  * i40e_get_link_speed - Returns link speed for the interface
5661  * @vsi: VSI to be configured
5662  *
5663  **/
5664 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5665 {
5666         struct i40e_pf *pf = vsi->back;
5667
5668         switch (pf->hw.phy.link_info.link_speed) {
5669         case I40E_LINK_SPEED_40GB:
5670                 return 40000;
5671         case I40E_LINK_SPEED_25GB:
5672                 return 25000;
5673         case I40E_LINK_SPEED_20GB:
5674                 return 20000;
5675         case I40E_LINK_SPEED_10GB:
5676                 return 10000;
5677         case I40E_LINK_SPEED_1GB:
5678                 return 1000;
5679         default:
5680                 return -EINVAL;
5681         }
5682 }
5683
5684 /**
5685  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5686  * @vsi: VSI to be configured
5687  * @seid: seid of the channel/VSI
5688  * @max_tx_rate: max TX rate to be configured as BW limit
5689  *
5690  * Helper function to set BW limit for a given VSI
5691  **/
5692 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5693 {
5694         struct i40e_pf *pf = vsi->back;
5695         u64 credits = 0;
5696         int speed = 0;
5697         int ret = 0;
5698
5699         speed = i40e_get_link_speed(vsi);
5700         if (max_tx_rate > speed) {
5701                 dev_err(&pf->pdev->dev,
5702                         "Invalid max tx rate %llu specified for VSI seid %d.",
5703                         max_tx_rate, seid);
5704                 return -EINVAL;
5705         }
5706         if (max_tx_rate && max_tx_rate < 50) {
5707                 dev_warn(&pf->pdev->dev,
5708                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5709                 max_tx_rate = 50;
5710         }
5711
5712         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5713         credits = max_tx_rate;
5714         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5715         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5716                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5717         if (ret)
5718                 dev_err(&pf->pdev->dev,
5719                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5720                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5721                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5722         return ret;
5723 }
5724
5725 /**
5726  * i40e_remove_queue_channels - Remove queue channels for the TCs
5727  * @vsi: VSI to be configured
5728  *
5729  * Remove queue channels for the TCs
5730  **/
5731 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5732 {
5733         enum i40e_admin_queue_err last_aq_status;
5734         struct i40e_cloud_filter *cfilter;
5735         struct i40e_channel *ch, *ch_tmp;
5736         struct i40e_pf *pf = vsi->back;
5737         struct hlist_node *node;
5738         int ret, i;
5739
5740         /* Reset rss size that was stored when reconfiguring rss for
5741          * channel VSIs with non-power-of-2 queue count.
5742          */
5743         vsi->current_rss_size = 0;
5744
5745         /* perform cleanup for channels if they exist */
5746         if (list_empty(&vsi->ch_list))
5747                 return;
5748
5749         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5750                 struct i40e_vsi *p_vsi;
5751
5752                 list_del(&ch->list);
5753                 p_vsi = ch->parent_vsi;
5754                 if (!p_vsi || !ch->initialized) {
5755                         kfree(ch);
5756                         continue;
5757                 }
5758                 /* Reset queue contexts */
5759                 for (i = 0; i < ch->num_queue_pairs; i++) {
5760                         struct i40e_ring *tx_ring, *rx_ring;
5761                         u16 pf_q;
5762
5763                         pf_q = ch->base_queue + i;
5764                         tx_ring = vsi->tx_rings[pf_q];
5765                         tx_ring->ch = NULL;
5766
5767                         rx_ring = vsi->rx_rings[pf_q];
5768                         rx_ring->ch = NULL;
5769                 }
5770
5771                 /* Reset BW configured for this VSI via mqprio */
5772                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5773                 if (ret)
5774                         dev_info(&vsi->back->pdev->dev,
5775                                  "Failed to reset tx rate for ch->seid %u\n",
5776                                  ch->seid);
5777
5778                 /* delete cloud filters associated with this channel */
5779                 hlist_for_each_entry_safe(cfilter, node,
5780                                           &pf->cloud_filter_list, cloud_node) {
5781                         if (cfilter->seid != ch->seid)
5782                                 continue;
5783
5784                         hash_del(&cfilter->cloud_node);
5785                         if (cfilter->dst_port)
5786                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5787                                                                         cfilter,
5788                                                                         false);
5789                         else
5790                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5791                                                                 false);
5792                         last_aq_status = pf->hw.aq.asq_last_status;
5793                         if (ret)
5794                                 dev_info(&pf->pdev->dev,
5795                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5796                                          i40e_stat_str(&pf->hw, ret),
5797                                          i40e_aq_str(&pf->hw, last_aq_status));
5798                         kfree(cfilter);
5799                 }
5800
5801                 /* delete VSI from FW */
5802                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5803                                              NULL);
5804                 if (ret)
5805                         dev_err(&vsi->back->pdev->dev,
5806                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5807                                 ch->seid, p_vsi->seid);
5808                 kfree(ch);
5809         }
5810         INIT_LIST_HEAD(&vsi->ch_list);
5811 }
5812
5813 /**
5814  * i40e_get_max_queues_for_channel
5815  * @vsi: ptr to VSI to which channels are associated with
5816  *
5817  * Helper function which returns max value among the queue counts set on the
5818  * channels/TCs created.
5819  **/
5820 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5821 {
5822         struct i40e_channel *ch, *ch_tmp;
5823         int max = 0;
5824
5825         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5826                 if (!ch->initialized)
5827                         continue;
5828                 if (ch->num_queue_pairs > max)
5829                         max = ch->num_queue_pairs;
5830         }
5831
5832         return max;
5833 }
5834
5835 /**
5836  * i40e_validate_num_queues - validate num_queues w.r.t channel
5837  * @pf: ptr to PF device
5838  * @num_queues: number of queues
5839  * @vsi: the parent VSI
5840  * @reconfig_rss: indicates should the RSS be reconfigured or not
5841  *
5842  * This function validates number of queues in the context of new channel
5843  * which is being established and determines if RSS should be reconfigured
5844  * or not for parent VSI.
5845  **/
5846 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5847                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5848 {
5849         int max_ch_queues;
5850
5851         if (!reconfig_rss)
5852                 return -EINVAL;
5853
5854         *reconfig_rss = false;
5855         if (vsi->current_rss_size) {
5856                 if (num_queues > vsi->current_rss_size) {
5857                         dev_dbg(&pf->pdev->dev,
5858                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5859                                 num_queues, vsi->current_rss_size);
5860                         return -EINVAL;
5861                 } else if ((num_queues < vsi->current_rss_size) &&
5862                            (!is_power_of_2(num_queues))) {
5863                         dev_dbg(&pf->pdev->dev,
5864                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5865                                 num_queues, vsi->current_rss_size);
5866                         return -EINVAL;
5867                 }
5868         }
5869
5870         if (!is_power_of_2(num_queues)) {
5871                 /* Find the max num_queues configured for channel if channel
5872                  * exist.
5873                  * if channel exist, then enforce 'num_queues' to be more than
5874                  * max ever queues configured for channel.
5875                  */
5876                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5877                 if (num_queues < max_ch_queues) {
5878                         dev_dbg(&pf->pdev->dev,
5879                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5880                                 num_queues, max_ch_queues);
5881                         return -EINVAL;
5882                 }
5883                 *reconfig_rss = true;
5884         }
5885
5886         return 0;
5887 }
5888
5889 /**
5890  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5891  * @vsi: the VSI being setup
5892  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5893  *
5894  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5895  **/
5896 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5897 {
5898         struct i40e_pf *pf = vsi->back;
5899         u8 seed[I40E_HKEY_ARRAY_SIZE];
5900         struct i40e_hw *hw = &pf->hw;
5901         int local_rss_size;
5902         u8 *lut;
5903         int ret;
5904
5905         if (!vsi->rss_size)
5906                 return -EINVAL;
5907
5908         if (rss_size > vsi->rss_size)
5909                 return -EINVAL;
5910
5911         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5912         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5913         if (!lut)
5914                 return -ENOMEM;
5915
5916         /* Ignoring user configured lut if there is one */
5917         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5918
5919         /* Use user configured hash key if there is one, otherwise
5920          * use default.
5921          */
5922         if (vsi->rss_hkey_user)
5923                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5924         else
5925                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5926
5927         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5928         if (ret) {
5929                 dev_info(&pf->pdev->dev,
5930                          "Cannot set RSS lut, err %s aq_err %s\n",
5931                          i40e_stat_str(hw, ret),
5932                          i40e_aq_str(hw, hw->aq.asq_last_status));
5933                 kfree(lut);
5934                 return ret;
5935         }
5936         kfree(lut);
5937
5938         /* Do the update w.r.t. storing rss_size */
5939         if (!vsi->orig_rss_size)
5940                 vsi->orig_rss_size = vsi->rss_size;
5941         vsi->current_rss_size = local_rss_size;
5942
5943         return ret;
5944 }
5945
5946 /**
5947  * i40e_channel_setup_queue_map - Setup a channel queue map
5948  * @pf: ptr to PF device
5949  * @ctxt: VSI context structure
5950  * @ch: ptr to channel structure
5951  *
5952  * Setup queue map for a specific channel
5953  **/
5954 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5955                                          struct i40e_vsi_context *ctxt,
5956                                          struct i40e_channel *ch)
5957 {
5958         u16 qcount, qmap, sections = 0;
5959         u8 offset = 0;
5960         int pow;
5961
5962         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5963         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5964
5965         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5966         ch->num_queue_pairs = qcount;
5967
5968         /* find the next higher power-of-2 of num queue pairs */
5969         pow = ilog2(qcount);
5970         if (!is_power_of_2(qcount))
5971                 pow++;
5972
5973         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5974                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5975
5976         /* Setup queue TC[0].qmap for given VSI context */
5977         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5978
5979         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5980         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5981         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5982         ctxt->info.valid_sections |= cpu_to_le16(sections);
5983 }
5984
5985 /**
5986  * i40e_add_channel - add a channel by adding VSI
5987  * @pf: ptr to PF device
5988  * @uplink_seid: underlying HW switching element (VEB) ID
5989  * @ch: ptr to channel structure
5990  *
5991  * Add a channel (VSI) using add_vsi and queue_map
5992  **/
5993 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5994                             struct i40e_channel *ch)
5995 {
5996         struct i40e_hw *hw = &pf->hw;
5997         struct i40e_vsi_context ctxt;
5998         u8 enabled_tc = 0x1; /* TC0 enabled */
5999         int ret;
6000
6001         if (ch->type != I40E_VSI_VMDQ2) {
6002                 dev_info(&pf->pdev->dev,
6003                          "add new vsi failed, ch->type %d\n", ch->type);
6004                 return -EINVAL;
6005         }
6006
6007         memset(&ctxt, 0, sizeof(ctxt));
6008         ctxt.pf_num = hw->pf_id;
6009         ctxt.vf_num = 0;
6010         ctxt.uplink_seid = uplink_seid;
6011         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6012         if (ch->type == I40E_VSI_VMDQ2)
6013                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6014
6015         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
6016                 ctxt.info.valid_sections |=
6017                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6018                 ctxt.info.switch_id =
6019                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6020         }
6021
6022         /* Set queue map for a given VSI context */
6023         i40e_channel_setup_queue_map(pf, &ctxt, ch);
6024
6025         /* Now time to create VSI */
6026         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6027         if (ret) {
6028                 dev_info(&pf->pdev->dev,
6029                          "add new vsi failed, err %s aq_err %s\n",
6030                          i40e_stat_str(&pf->hw, ret),
6031                          i40e_aq_str(&pf->hw,
6032                                      pf->hw.aq.asq_last_status));
6033                 return -ENOENT;
6034         }
6035
6036         /* Success, update channel, set enabled_tc only if the channel
6037          * is not a macvlan
6038          */
6039         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6040         ch->seid = ctxt.seid;
6041         ch->vsi_number = ctxt.vsi_number;
6042         ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6043
6044         /* copy just the sections touched not the entire info
6045          * since not all sections are valid as returned by
6046          * update vsi params
6047          */
6048         ch->info.mapping_flags = ctxt.info.mapping_flags;
6049         memcpy(&ch->info.queue_mapping,
6050                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6051         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6052                sizeof(ctxt.info.tc_mapping));
6053
6054         return 0;
6055 }
6056
6057 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6058                                   u8 *bw_share)
6059 {
6060         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6061         i40e_status ret;
6062         int i;
6063
6064         memset(&bw_data, 0, sizeof(bw_data));
6065         bw_data.tc_valid_bits = ch->enabled_tc;
6066         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6067                 bw_data.tc_bw_credits[i] = bw_share[i];
6068
6069         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6070                                        &bw_data, NULL);
6071         if (ret) {
6072                 dev_info(&vsi->back->pdev->dev,
6073                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6074                          vsi->back->hw.aq.asq_last_status, ch->seid);
6075                 return -EINVAL;
6076         }
6077
6078         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6079                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6080
6081         return 0;
6082 }
6083
6084 /**
6085  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6086  * @pf: ptr to PF device
6087  * @vsi: the VSI being setup
6088  * @ch: ptr to channel structure
6089  *
6090  * Configure TX rings associated with channel (VSI) since queues are being
6091  * from parent VSI.
6092  **/
6093 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6094                                        struct i40e_vsi *vsi,
6095                                        struct i40e_channel *ch)
6096 {
6097         i40e_status ret;
6098         int i;
6099         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6100
6101         /* Enable ETS TCs with equal BW Share for now across all VSIs */
6102         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6103                 if (ch->enabled_tc & BIT(i))
6104                         bw_share[i] = 1;
6105         }
6106
6107         /* configure BW for new VSI */
6108         ret = i40e_channel_config_bw(vsi, ch, bw_share);
6109         if (ret) {
6110                 dev_info(&vsi->back->pdev->dev,
6111                          "Failed configuring TC map %d for channel (seid %u)\n",
6112                          ch->enabled_tc, ch->seid);
6113                 return ret;
6114         }
6115
6116         for (i = 0; i < ch->num_queue_pairs; i++) {
6117                 struct i40e_ring *tx_ring, *rx_ring;
6118                 u16 pf_q;
6119
6120                 pf_q = ch->base_queue + i;
6121
6122                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6123                  * context
6124                  */
6125                 tx_ring = vsi->tx_rings[pf_q];
6126                 tx_ring->ch = ch;
6127
6128                 /* Get the RX ring ptr */
6129                 rx_ring = vsi->rx_rings[pf_q];
6130                 rx_ring->ch = ch;
6131         }
6132
6133         return 0;
6134 }
6135
6136 /**
6137  * i40e_setup_hw_channel - setup new channel
6138  * @pf: ptr to PF device
6139  * @vsi: the VSI being setup
6140  * @ch: ptr to channel structure
6141  * @uplink_seid: underlying HW switching element (VEB) ID
6142  * @type: type of channel to be created (VMDq2/VF)
6143  *
6144  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6145  * and configures TX rings accordingly
6146  **/
6147 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6148                                         struct i40e_vsi *vsi,
6149                                         struct i40e_channel *ch,
6150                                         u16 uplink_seid, u8 type)
6151 {
6152         int ret;
6153
6154         ch->initialized = false;
6155         ch->base_queue = vsi->next_base_queue;
6156         ch->type = type;
6157
6158         /* Proceed with creation of channel (VMDq2) VSI */
6159         ret = i40e_add_channel(pf, uplink_seid, ch);
6160         if (ret) {
6161                 dev_info(&pf->pdev->dev,
6162                          "failed to add_channel using uplink_seid %u\n",
6163                          uplink_seid);
6164                 return ret;
6165         }
6166
6167         /* Mark the successful creation of channel */
6168         ch->initialized = true;
6169
6170         /* Reconfigure TX queues using QTX_CTL register */
6171         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6172         if (ret) {
6173                 dev_info(&pf->pdev->dev,
6174                          "failed to configure TX rings for channel %u\n",
6175                          ch->seid);
6176                 return ret;
6177         }
6178
6179         /* update 'next_base_queue' */
6180         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6181         dev_dbg(&pf->pdev->dev,
6182                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6183                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6184                 ch->num_queue_pairs,
6185                 vsi->next_base_queue);
6186         return ret;
6187 }
6188
6189 /**
6190  * i40e_setup_channel - setup new channel using uplink element
6191  * @pf: ptr to PF device
6192  * @vsi: pointer to the VSI to set up the channel within
6193  * @ch: ptr to channel structure
6194  *
6195  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6196  * and uplink switching element (uplink_seid)
6197  **/
6198 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6199                                struct i40e_channel *ch)
6200 {
6201         u8 vsi_type;
6202         u16 seid;
6203         int ret;
6204
6205         if (vsi->type == I40E_VSI_MAIN) {
6206                 vsi_type = I40E_VSI_VMDQ2;
6207         } else {
6208                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6209                         vsi->type);
6210                 return false;
6211         }
6212
6213         /* underlying switching element */
6214         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6215
6216         /* create channel (VSI), configure TX rings */
6217         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6218         if (ret) {
6219                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6220                 return false;
6221         }
6222
6223         return ch->initialized ? true : false;
6224 }
6225
6226 /**
6227  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6228  * @vsi: ptr to VSI which has PF backing
6229  *
6230  * Sets up switch mode correctly if it needs to be changed and perform
6231  * what are allowed modes.
6232  **/
6233 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6234 {
6235         u8 mode;
6236         struct i40e_pf *pf = vsi->back;
6237         struct i40e_hw *hw = &pf->hw;
6238         int ret;
6239
6240         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6241         if (ret)
6242                 return -EINVAL;
6243
6244         if (hw->dev_caps.switch_mode) {
6245                 /* if switch mode is set, support mode2 (non-tunneled for
6246                  * cloud filter) for now
6247                  */
6248                 u32 switch_mode = hw->dev_caps.switch_mode &
6249                                   I40E_SWITCH_MODE_MASK;
6250                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6251                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6252                                 return 0;
6253                         dev_err(&pf->pdev->dev,
6254                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6255                                 hw->dev_caps.switch_mode);
6256                         return -EINVAL;
6257                 }
6258         }
6259
6260         /* Set Bit 7 to be valid */
6261         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6262
6263         /* Set L4type for TCP support */
6264         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6265
6266         /* Set cloud filter mode */
6267         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6268
6269         /* Prep mode field for set_switch_config */
6270         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6271                                         pf->last_sw_conf_valid_flags,
6272                                         mode, NULL);
6273         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6274                 dev_err(&pf->pdev->dev,
6275                         "couldn't set switch config bits, err %s aq_err %s\n",
6276                         i40e_stat_str(hw, ret),
6277                         i40e_aq_str(hw,
6278                                     hw->aq.asq_last_status));
6279
6280         return ret;
6281 }
6282
6283 /**
6284  * i40e_create_queue_channel - function to create channel
6285  * @vsi: VSI to be configured
6286  * @ch: ptr to channel (it contains channel specific params)
6287  *
6288  * This function creates channel (VSI) using num_queues specified by user,
6289  * reconfigs RSS if needed.
6290  **/
6291 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6292                               struct i40e_channel *ch)
6293 {
6294         struct i40e_pf *pf = vsi->back;
6295         bool reconfig_rss;
6296         int err;
6297
6298         if (!ch)
6299                 return -EINVAL;
6300
6301         if (!ch->num_queue_pairs) {
6302                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6303                         ch->num_queue_pairs);
6304                 return -EINVAL;
6305         }
6306
6307         /* validate user requested num_queues for channel */
6308         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6309                                        &reconfig_rss);
6310         if (err) {
6311                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6312                          ch->num_queue_pairs);
6313                 return -EINVAL;
6314         }
6315
6316         /* By default we are in VEPA mode, if this is the first VF/VMDq
6317          * VSI to be added switch to VEB mode.
6318          */
6319
6320         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6321                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6322
6323                 if (vsi->type == I40E_VSI_MAIN) {
6324                         if (pf->flags & I40E_FLAG_TC_MQPRIO)
6325                                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6326                         else
6327                                 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6328                 }
6329                 /* now onwards for main VSI, number of queues will be value
6330                  * of TC0's queue count
6331                  */
6332         }
6333
6334         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6335          * it should be more than num_queues
6336          */
6337         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6338                 dev_dbg(&pf->pdev->dev,
6339                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6340                         vsi->cnt_q_avail, ch->num_queue_pairs);
6341                 return -EINVAL;
6342         }
6343
6344         /* reconfig_rss only if vsi type is MAIN_VSI */
6345         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6346                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6347                 if (err) {
6348                         dev_info(&pf->pdev->dev,
6349                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6350                                  ch->num_queue_pairs);
6351                         return -EINVAL;
6352                 }
6353         }
6354
6355         if (!i40e_setup_channel(pf, vsi, ch)) {
6356                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6357                 return -EINVAL;
6358         }
6359
6360         dev_info(&pf->pdev->dev,
6361                  "Setup channel (id:%u) utilizing num_queues %d\n",
6362                  ch->seid, ch->num_queue_pairs);
6363
6364         /* configure VSI for BW limit */
6365         if (ch->max_tx_rate) {
6366                 u64 credits = ch->max_tx_rate;
6367
6368                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6369                         return -EINVAL;
6370
6371                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6372                 dev_dbg(&pf->pdev->dev,
6373                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6374                         ch->max_tx_rate,
6375                         credits,
6376                         ch->seid);
6377         }
6378
6379         /* in case of VF, this will be main SRIOV VSI */
6380         ch->parent_vsi = vsi;
6381
6382         /* and update main_vsi's count for queue_available to use */
6383         vsi->cnt_q_avail -= ch->num_queue_pairs;
6384
6385         return 0;
6386 }
6387
6388 /**
6389  * i40e_configure_queue_channels - Add queue channel for the given TCs
6390  * @vsi: VSI to be configured
6391  *
6392  * Configures queue channel mapping to the given TCs
6393  **/
6394 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6395 {
6396         struct i40e_channel *ch;
6397         u64 max_rate = 0;
6398         int ret = 0, i;
6399
6400         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6401         vsi->tc_seid_map[0] = vsi->seid;
6402         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6403                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6404                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6405                         if (!ch) {
6406                                 ret = -ENOMEM;
6407                                 goto err_free;
6408                         }
6409
6410                         INIT_LIST_HEAD(&ch->list);
6411                         ch->num_queue_pairs =
6412                                 vsi->tc_config.tc_info[i].qcount;
6413                         ch->base_queue =
6414                                 vsi->tc_config.tc_info[i].qoffset;
6415
6416                         /* Bandwidth limit through tc interface is in bytes/s,
6417                          * change to Mbit/s
6418                          */
6419                         max_rate = vsi->mqprio_qopt.max_rate[i];
6420                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6421                         ch->max_tx_rate = max_rate;
6422
6423                         list_add_tail(&ch->list, &vsi->ch_list);
6424
6425                         ret = i40e_create_queue_channel(vsi, ch);
6426                         if (ret) {
6427                                 dev_err(&vsi->back->pdev->dev,
6428                                         "Failed creating queue channel with TC%d: queues %d\n",
6429                                         i, ch->num_queue_pairs);
6430                                 goto err_free;
6431                         }
6432                         vsi->tc_seid_map[i] = ch->seid;
6433                 }
6434         }
6435         return ret;
6436
6437 err_free:
6438         i40e_remove_queue_channels(vsi);
6439         return ret;
6440 }
6441
6442 /**
6443  * i40e_veb_config_tc - Configure TCs for given VEB
6444  * @veb: given VEB
6445  * @enabled_tc: TC bitmap
6446  *
6447  * Configures given TC bitmap for VEB (switching) element
6448  **/
6449 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6450 {
6451         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6452         struct i40e_pf *pf = veb->pf;
6453         int ret = 0;
6454         int i;
6455
6456         /* No TCs or already enabled TCs just return */
6457         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6458                 return ret;
6459
6460         bw_data.tc_valid_bits = enabled_tc;
6461         /* bw_data.absolute_credits is not set (relative) */
6462
6463         /* Enable ETS TCs with equal BW Share for now */
6464         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6465                 if (enabled_tc & BIT(i))
6466                         bw_data.tc_bw_share_credits[i] = 1;
6467         }
6468
6469         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6470                                                    &bw_data, NULL);
6471         if (ret) {
6472                 dev_info(&pf->pdev->dev,
6473                          "VEB bw config failed, err %s aq_err %s\n",
6474                          i40e_stat_str(&pf->hw, ret),
6475                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6476                 goto out;
6477         }
6478
6479         /* Update the BW information */
6480         ret = i40e_veb_get_bw_info(veb);
6481         if (ret) {
6482                 dev_info(&pf->pdev->dev,
6483                          "Failed getting veb bw config, err %s aq_err %s\n",
6484                          i40e_stat_str(&pf->hw, ret),
6485                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6486         }
6487
6488 out:
6489         return ret;
6490 }
6491
6492 #ifdef CONFIG_I40E_DCB
6493 /**
6494  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6495  * @pf: PF struct
6496  *
6497  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6498  * the caller would've quiesce all the VSIs before calling
6499  * this function
6500  **/
6501 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6502 {
6503         u8 tc_map = 0;
6504         int ret;
6505         u8 v;
6506
6507         /* Enable the TCs available on PF to all VEBs */
6508         tc_map = i40e_pf_get_tc_map(pf);
6509         if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6510                 return;
6511
6512         for (v = 0; v < I40E_MAX_VEB; v++) {
6513                 if (!pf->veb[v])
6514                         continue;
6515                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6516                 if (ret) {
6517                         dev_info(&pf->pdev->dev,
6518                                  "Failed configuring TC for VEB seid=%d\n",
6519                                  pf->veb[v]->seid);
6520                         /* Will try to configure as many components */
6521                 }
6522         }
6523
6524         /* Update each VSI */
6525         for (v = 0; v < pf->num_alloc_vsi; v++) {
6526                 if (!pf->vsi[v])
6527                         continue;
6528
6529                 /* - Enable all TCs for the LAN VSI
6530                  * - For all others keep them at TC0 for now
6531                  */
6532                 if (v == pf->lan_vsi)
6533                         tc_map = i40e_pf_get_tc_map(pf);
6534                 else
6535                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6536
6537                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6538                 if (ret) {
6539                         dev_info(&pf->pdev->dev,
6540                                  "Failed configuring TC for VSI seid=%d\n",
6541                                  pf->vsi[v]->seid);
6542                         /* Will try to configure as many components */
6543                 } else {
6544                         /* Re-configure VSI vectors based on updated TC map */
6545                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6546                         if (pf->vsi[v]->netdev)
6547                                 i40e_dcbnl_set_all(pf->vsi[v]);
6548                 }
6549         }
6550 }
6551
6552 /**
6553  * i40e_resume_port_tx - Resume port Tx
6554  * @pf: PF struct
6555  *
6556  * Resume a port's Tx and issue a PF reset in case of failure to
6557  * resume.
6558  **/
6559 static int i40e_resume_port_tx(struct i40e_pf *pf)
6560 {
6561         struct i40e_hw *hw = &pf->hw;
6562         int ret;
6563
6564         ret = i40e_aq_resume_port_tx(hw, NULL);
6565         if (ret) {
6566                 dev_info(&pf->pdev->dev,
6567                          "Resume Port Tx failed, err %s aq_err %s\n",
6568                           i40e_stat_str(&pf->hw, ret),
6569                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6570                 /* Schedule PF reset to recover */
6571                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6572                 i40e_service_event_schedule(pf);
6573         }
6574
6575         return ret;
6576 }
6577
6578 /**
6579  * i40e_suspend_port_tx - Suspend port Tx
6580  * @pf: PF struct
6581  *
6582  * Suspend a port's Tx and issue a PF reset in case of failure.
6583  **/
6584 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6585 {
6586         struct i40e_hw *hw = &pf->hw;
6587         int ret;
6588
6589         ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6590         if (ret) {
6591                 dev_info(&pf->pdev->dev,
6592                          "Suspend Port Tx failed, err %s aq_err %s\n",
6593                          i40e_stat_str(&pf->hw, ret),
6594                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6595                 /* Schedule PF reset to recover */
6596                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6597                 i40e_service_event_schedule(pf);
6598         }
6599
6600         return ret;
6601 }
6602
6603 /**
6604  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6605  * @pf: PF being configured
6606  * @new_cfg: New DCBX configuration
6607  *
6608  * Program DCB settings into HW and reconfigure VEB/VSIs on
6609  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6610  **/
6611 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6612                                   struct i40e_dcbx_config *new_cfg)
6613 {
6614         struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6615         int ret;
6616
6617         /* Check if need reconfiguration */
6618         if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6619                 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6620                 return 0;
6621         }
6622
6623         /* Config change disable all VSIs */
6624         i40e_pf_quiesce_all_vsi(pf);
6625
6626         /* Copy the new config to the current config */
6627         *old_cfg = *new_cfg;
6628         old_cfg->etsrec = old_cfg->etscfg;
6629         ret = i40e_set_dcb_config(&pf->hw);
6630         if (ret) {
6631                 dev_info(&pf->pdev->dev,
6632                          "Set DCB Config failed, err %s aq_err %s\n",
6633                          i40e_stat_str(&pf->hw, ret),
6634                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6635                 goto out;
6636         }
6637
6638         /* Changes in configuration update VEB/VSI */
6639         i40e_dcb_reconfigure(pf);
6640 out:
6641         /* In case of reset do not try to resume anything */
6642         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6643                 /* Re-start the VSIs if disabled */
6644                 ret = i40e_resume_port_tx(pf);
6645                 /* In case of error no point in resuming VSIs */
6646                 if (ret)
6647                         goto err;
6648                 i40e_pf_unquiesce_all_vsi(pf);
6649         }
6650 err:
6651         return ret;
6652 }
6653
6654 /**
6655  * i40e_hw_dcb_config - Program new DCBX settings into HW
6656  * @pf: PF being configured
6657  * @new_cfg: New DCBX configuration
6658  *
6659  * Program DCB settings into HW and reconfigure VEB/VSIs on
6660  * given PF
6661  **/
6662 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6663 {
6664         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6665         u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6666         u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6667         struct i40e_dcbx_config *old_cfg;
6668         u8 mode[I40E_MAX_TRAFFIC_CLASS];
6669         struct i40e_rx_pb_config pb_cfg;
6670         struct i40e_hw *hw = &pf->hw;
6671         u8 num_ports = hw->num_ports;
6672         bool need_reconfig;
6673         int ret = -EINVAL;
6674         u8 lltc_map = 0;
6675         u8 tc_map = 0;
6676         u8 new_numtc;
6677         u8 i;
6678
6679         dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6680         /* Un-pack information to Program ETS HW via shared API
6681          * numtc, tcmap
6682          * LLTC map
6683          * ETS/NON-ETS arbiter mode
6684          * max exponent (credit refills)
6685          * Total number of ports
6686          * PFC priority bit-map
6687          * Priority Table
6688          * BW % per TC
6689          * Arbiter mode between UPs sharing same TC
6690          * TSA table (ETS or non-ETS)
6691          * EEE enabled or not
6692          * MFS TC table
6693          */
6694
6695         new_numtc = i40e_dcb_get_num_tc(new_cfg);
6696
6697         memset(&ets_data, 0, sizeof(ets_data));
6698         for (i = 0; i < new_numtc; i++) {
6699                 tc_map |= BIT(i);
6700                 switch (new_cfg->etscfg.tsatable[i]) {
6701                 case I40E_IEEE_TSA_ETS:
6702                         prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6703                         ets_data.tc_bw_share_credits[i] =
6704                                         new_cfg->etscfg.tcbwtable[i];
6705                         break;
6706                 case I40E_IEEE_TSA_STRICT:
6707                         prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6708                         lltc_map |= BIT(i);
6709                         ets_data.tc_bw_share_credits[i] =
6710                                         I40E_DCB_STRICT_PRIO_CREDITS;
6711                         break;
6712                 default:
6713                         /* Invalid TSA type */
6714                         need_reconfig = false;
6715                         goto out;
6716                 }
6717         }
6718
6719         old_cfg = &hw->local_dcbx_config;
6720         /* Check if need reconfiguration */
6721         need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
6722
6723         /* If needed, enable/disable frame tagging, disable all VSIs
6724          * and suspend port tx
6725          */
6726         if (need_reconfig) {
6727                 /* Enable DCB tagging only when more than one TC */
6728                 if (new_numtc > 1)
6729                         pf->flags |= I40E_FLAG_DCB_ENABLED;
6730                 else
6731                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6732
6733                 set_bit(__I40E_PORT_SUSPENDED, pf->state);
6734                 /* Reconfiguration needed quiesce all VSIs */
6735                 i40e_pf_quiesce_all_vsi(pf);
6736                 ret = i40e_suspend_port_tx(pf);
6737                 if (ret)
6738                         goto err;
6739         }
6740
6741         /* Configure Port ETS Tx Scheduler */
6742         ets_data.tc_valid_bits = tc_map;
6743         ets_data.tc_strict_priority_flags = lltc_map;
6744         ret = i40e_aq_config_switch_comp_ets
6745                 (hw, pf->mac_seid, &ets_data,
6746                  i40e_aqc_opc_modify_switching_comp_ets, NULL);
6747         if (ret) {
6748                 dev_info(&pf->pdev->dev,
6749                          "Modify Port ETS failed, err %s aq_err %s\n",
6750                          i40e_stat_str(&pf->hw, ret),
6751                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6752                 goto out;
6753         }
6754
6755         /* Configure Rx ETS HW */
6756         memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
6757         i40e_dcb_hw_set_num_tc(hw, new_numtc);
6758         i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
6759                                    I40E_DCB_ARB_MODE_STRICT_PRIORITY,
6760                                    I40E_DCB_DEFAULT_MAX_EXPONENT,
6761                                    lltc_map);
6762         i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
6763         i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
6764                                      prio_type);
6765         i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
6766                                new_cfg->etscfg.prioritytable);
6767         i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
6768
6769         /* Configure Rx Packet Buffers in HW */
6770         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6771                 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
6772                 mfs_tc[i] += I40E_PACKET_HDR_PAD;
6773         }
6774
6775         i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
6776                                          false, new_cfg->pfc.pfcenable,
6777                                          mfs_tc, &pb_cfg);
6778         i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
6779
6780         /* Update the local Rx Packet buffer config */
6781         pf->pb_cfg = pb_cfg;
6782
6783         /* Inform the FW about changes to DCB configuration */
6784         ret = i40e_aq_dcb_updated(&pf->hw, NULL);
6785         if (ret) {
6786                 dev_info(&pf->pdev->dev,
6787                          "DCB Updated failed, err %s aq_err %s\n",
6788                          i40e_stat_str(&pf->hw, ret),
6789                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6790                 goto out;
6791         }
6792
6793         /* Update the port DCBx configuration */
6794         *old_cfg = *new_cfg;
6795
6796         /* Changes in configuration update VEB/VSI */
6797         i40e_dcb_reconfigure(pf);
6798 out:
6799         /* Re-start the VSIs if disabled */
6800         if (need_reconfig) {
6801                 ret = i40e_resume_port_tx(pf);
6802
6803                 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6804                 /* In case of error no point in resuming VSIs */
6805                 if (ret)
6806                         goto err;
6807
6808                 /* Wait for the PF's queues to be disabled */
6809                 ret = i40e_pf_wait_queues_disabled(pf);
6810                 if (ret) {
6811                         /* Schedule PF reset to recover */
6812                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6813                         i40e_service_event_schedule(pf);
6814                         goto err;
6815                 } else {
6816                         i40e_pf_unquiesce_all_vsi(pf);
6817                         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6818                         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
6819                 }
6820                 /* registers are set, lets apply */
6821                 if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
6822                         ret = i40e_hw_set_dcb_config(pf, new_cfg);
6823         }
6824
6825 err:
6826         return ret;
6827 }
6828
6829 /**
6830  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
6831  * @pf: PF being queried
6832  *
6833  * Set default DCB configuration in case DCB is to be done in SW.
6834  **/
6835 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
6836 {
6837         struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
6838         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6839         struct i40e_hw *hw = &pf->hw;
6840         int err;
6841
6842         if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
6843                 /* Update the local cached instance with TC0 ETS */
6844                 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
6845                 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6846                 pf->tmp_cfg.etscfg.maxtcs = 0;
6847                 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6848                 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
6849                 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
6850                 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
6851                 /* FW needs one App to configure HW */
6852                 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
6853                 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
6854                 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
6855                 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
6856
6857                 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
6858         }
6859
6860         memset(&ets_data, 0, sizeof(ets_data));
6861         ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
6862         ets_data.tc_strict_priority_flags = 0; /* ETS */
6863         ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
6864
6865         /* Enable ETS on the Physical port */
6866         err = i40e_aq_config_switch_comp_ets
6867                 (hw, pf->mac_seid, &ets_data,
6868                  i40e_aqc_opc_enable_switching_comp_ets, NULL);
6869         if (err) {
6870                 dev_info(&pf->pdev->dev,
6871                          "Enable Port ETS failed, err %s aq_err %s\n",
6872                          i40e_stat_str(&pf->hw, err),
6873                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6874                 err = -ENOENT;
6875                 goto out;
6876         }
6877
6878         /* Update the local cached instance with TC0 ETS */
6879         dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6880         dcb_cfg->etscfg.cbs = 0;
6881         dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
6882         dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6883
6884 out:
6885         return err;
6886 }
6887
6888 /**
6889  * i40e_init_pf_dcb - Initialize DCB configuration
6890  * @pf: PF being configured
6891  *
6892  * Query the current DCB configuration and cache it
6893  * in the hardware structure
6894  **/
6895 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6896 {
6897         struct i40e_hw *hw = &pf->hw;
6898         int err;
6899
6900         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6901          * Also do not enable DCBx if FW LLDP agent is disabled
6902          */
6903         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
6904                 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
6905                 err = I40E_NOT_SUPPORTED;
6906                 goto out;
6907         }
6908         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
6909                 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
6910                 err = i40e_dcb_sw_default_config(pf);
6911                 if (err) {
6912                         dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
6913                         goto out;
6914                 }
6915                 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
6916                 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
6917                                DCB_CAP_DCBX_VER_IEEE;
6918                 /* at init capable but disabled */
6919                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
6920                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6921                 goto out;
6922         }
6923         err = i40e_init_dcb(hw, true);
6924         if (!err) {
6925                 /* Device/Function is not DCBX capable */
6926                 if ((!hw->func_caps.dcb) ||
6927                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6928                         dev_info(&pf->pdev->dev,
6929                                  "DCBX offload is not supported or is disabled for this PF.\n");
6930                 } else {
6931                         /* When status is not DISABLED then DCBX in FW */
6932                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6933                                        DCB_CAP_DCBX_VER_IEEE;
6934
6935                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6936                         /* Enable DCB tagging only when more than one TC
6937                          * or explicitly disable if only one TC
6938                          */
6939                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6940                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6941                         else
6942                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6943                         dev_dbg(&pf->pdev->dev,
6944                                 "DCBX offload is supported for this PF.\n");
6945                 }
6946         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6947                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6948                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6949         } else {
6950                 dev_info(&pf->pdev->dev,
6951                          "Query for DCB configuration failed, err %s aq_err %s\n",
6952                          i40e_stat_str(&pf->hw, err),
6953                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6954         }
6955
6956 out:
6957         return err;
6958 }
6959 #endif /* CONFIG_I40E_DCB */
6960
6961 /**
6962  * i40e_print_link_message - print link up or down
6963  * @vsi: the VSI for which link needs a message
6964  * @isup: true of link is up, false otherwise
6965  */
6966 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6967 {
6968         enum i40e_aq_link_speed new_speed;
6969         struct i40e_pf *pf = vsi->back;
6970         char *speed = "Unknown";
6971         char *fc = "Unknown";
6972         char *fec = "";
6973         char *req_fec = "";
6974         char *an = "";
6975
6976         if (isup)
6977                 new_speed = pf->hw.phy.link_info.link_speed;
6978         else
6979                 new_speed = I40E_LINK_SPEED_UNKNOWN;
6980
6981         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6982                 return;
6983         vsi->current_isup = isup;
6984         vsi->current_speed = new_speed;
6985         if (!isup) {
6986                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6987                 return;
6988         }
6989
6990         /* Warn user if link speed on NPAR enabled partition is not at
6991          * least 10GB
6992          */
6993         if (pf->hw.func_caps.npar_enable &&
6994             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6995              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6996                 netdev_warn(vsi->netdev,
6997                             "The partition detected link speed that is less than 10Gbps\n");
6998
6999         switch (pf->hw.phy.link_info.link_speed) {
7000         case I40E_LINK_SPEED_40GB:
7001                 speed = "40 G";
7002                 break;
7003         case I40E_LINK_SPEED_20GB:
7004                 speed = "20 G";
7005                 break;
7006         case I40E_LINK_SPEED_25GB:
7007                 speed = "25 G";
7008                 break;
7009         case I40E_LINK_SPEED_10GB:
7010                 speed = "10 G";
7011                 break;
7012         case I40E_LINK_SPEED_5GB:
7013                 speed = "5 G";
7014                 break;
7015         case I40E_LINK_SPEED_2_5GB:
7016                 speed = "2.5 G";
7017                 break;
7018         case I40E_LINK_SPEED_1GB:
7019                 speed = "1000 M";
7020                 break;
7021         case I40E_LINK_SPEED_100MB:
7022                 speed = "100 M";
7023                 break;
7024         default:
7025                 break;
7026         }
7027
7028         switch (pf->hw.fc.current_mode) {
7029         case I40E_FC_FULL:
7030                 fc = "RX/TX";
7031                 break;
7032         case I40E_FC_TX_PAUSE:
7033                 fc = "TX";
7034                 break;
7035         case I40E_FC_RX_PAUSE:
7036                 fc = "RX";
7037                 break;
7038         default:
7039                 fc = "None";
7040                 break;
7041         }
7042
7043         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7044                 req_fec = "None";
7045                 fec = "None";
7046                 an = "False";
7047
7048                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7049                         an = "True";
7050
7051                 if (pf->hw.phy.link_info.fec_info &
7052                     I40E_AQ_CONFIG_FEC_KR_ENA)
7053                         fec = "CL74 FC-FEC/BASE-R";
7054                 else if (pf->hw.phy.link_info.fec_info &
7055                          I40E_AQ_CONFIG_FEC_RS_ENA)
7056                         fec = "CL108 RS-FEC";
7057
7058                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7059                  * both RS and FC are requested
7060                  */
7061                 if (vsi->back->hw.phy.link_info.req_fec_info &
7062                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7063                         if (vsi->back->hw.phy.link_info.req_fec_info &
7064                             I40E_AQ_REQUEST_FEC_RS)
7065                                 req_fec = "CL108 RS-FEC";
7066                         else
7067                                 req_fec = "CL74 FC-FEC/BASE-R";
7068                 }
7069                 netdev_info(vsi->netdev,
7070                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7071                             speed, req_fec, fec, an, fc);
7072         } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7073                 req_fec = "None";
7074                 fec = "None";
7075                 an = "False";
7076
7077                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7078                         an = "True";
7079
7080                 if (pf->hw.phy.link_info.fec_info &
7081                     I40E_AQ_CONFIG_FEC_KR_ENA)
7082                         fec = "CL74 FC-FEC/BASE-R";
7083
7084                 if (pf->hw.phy.link_info.req_fec_info &
7085                     I40E_AQ_REQUEST_FEC_KR)
7086                         req_fec = "CL74 FC-FEC/BASE-R";
7087
7088                 netdev_info(vsi->netdev,
7089                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7090                             speed, req_fec, fec, an, fc);
7091         } else {
7092                 netdev_info(vsi->netdev,
7093                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7094                             speed, fc);
7095         }
7096
7097 }
7098
7099 /**
7100  * i40e_up_complete - Finish the last steps of bringing up a connection
7101  * @vsi: the VSI being configured
7102  **/
7103 static int i40e_up_complete(struct i40e_vsi *vsi)
7104 {
7105         struct i40e_pf *pf = vsi->back;
7106         int err;
7107
7108         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7109                 i40e_vsi_configure_msix(vsi);
7110         else
7111                 i40e_configure_msi_and_legacy(vsi);
7112
7113         /* start rings */
7114         err = i40e_vsi_start_rings(vsi);
7115         if (err)
7116                 return err;
7117
7118         clear_bit(__I40E_VSI_DOWN, vsi->state);
7119         i40e_napi_enable_all(vsi);
7120         i40e_vsi_enable_irq(vsi);
7121
7122         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7123             (vsi->netdev)) {
7124                 i40e_print_link_message(vsi, true);
7125                 netif_tx_start_all_queues(vsi->netdev);
7126                 netif_carrier_on(vsi->netdev);
7127         }
7128
7129         /* replay FDIR SB filters */
7130         if (vsi->type == I40E_VSI_FDIR) {
7131                 /* reset fd counters */
7132                 pf->fd_add_err = 0;
7133                 pf->fd_atr_cnt = 0;
7134                 i40e_fdir_filter_restore(vsi);
7135         }
7136
7137         /* On the next run of the service_task, notify any clients of the new
7138          * opened netdev
7139          */
7140         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7141         i40e_service_event_schedule(pf);
7142
7143         return 0;
7144 }
7145
7146 /**
7147  * i40e_vsi_reinit_locked - Reset the VSI
7148  * @vsi: the VSI being configured
7149  *
7150  * Rebuild the ring structs after some configuration
7151  * has changed, e.g. MTU size.
7152  **/
7153 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7154 {
7155         struct i40e_pf *pf = vsi->back;
7156
7157         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7158                 usleep_range(1000, 2000);
7159         i40e_down(vsi);
7160
7161         i40e_up(vsi);
7162         clear_bit(__I40E_CONFIG_BUSY, pf->state);
7163 }
7164
7165 /**
7166  * i40e_force_link_state - Force the link status
7167  * @pf: board private structure
7168  * @is_up: whether the link state should be forced up or down
7169  **/
7170 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7171 {
7172         struct i40e_aq_get_phy_abilities_resp abilities;
7173         struct i40e_aq_set_phy_config config = {0};
7174         bool non_zero_phy_type = is_up;
7175         struct i40e_hw *hw = &pf->hw;
7176         i40e_status err;
7177         u64 mask;
7178         u8 speed;
7179
7180         /* Card might've been put in an unstable state by other drivers
7181          * and applications, which causes incorrect speed values being
7182          * set on startup. In order to clear speed registers, we call
7183          * get_phy_capabilities twice, once to get initial state of
7184          * available speeds, and once to get current PHY config.
7185          */
7186         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7187                                            NULL);
7188         if (err) {
7189                 dev_err(&pf->pdev->dev,
7190                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7191                         i40e_stat_str(hw, err),
7192                         i40e_aq_str(hw, hw->aq.asq_last_status));
7193                 return err;
7194         }
7195         speed = abilities.link_speed;
7196
7197         /* Get the current phy config */
7198         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7199                                            NULL);
7200         if (err) {
7201                 dev_err(&pf->pdev->dev,
7202                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7203                         i40e_stat_str(hw, err),
7204                         i40e_aq_str(hw, hw->aq.asq_last_status));
7205                 return err;
7206         }
7207
7208         /* If link needs to go up, but was not forced to go down,
7209          * and its speed values are OK, no need for a flap
7210          * if non_zero_phy_type was set, still need to force up
7211          */
7212         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
7213                 non_zero_phy_type = true;
7214         else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7215                 return I40E_SUCCESS;
7216
7217         /* To force link we need to set bits for all supported PHY types,
7218          * but there are now more than 32, so we need to split the bitmap
7219          * across two fields.
7220          */
7221         mask = I40E_PHY_TYPES_BITMASK;
7222         config.phy_type =
7223                 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7224         config.phy_type_ext =
7225                 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7226         /* Copy the old settings, except of phy_type */
7227         config.abilities = abilities.abilities;
7228         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
7229                 if (is_up)
7230                         config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7231                 else
7232                         config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7233         }
7234         if (abilities.link_speed != 0)
7235                 config.link_speed = abilities.link_speed;
7236         else
7237                 config.link_speed = speed;
7238         config.eee_capability = abilities.eee_capability;
7239         config.eeer = abilities.eeer_val;
7240         config.low_power_ctrl = abilities.d3_lpan;
7241         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7242                             I40E_AQ_PHY_FEC_CONFIG_MASK;
7243         err = i40e_aq_set_phy_config(hw, &config, NULL);
7244
7245         if (err) {
7246                 dev_err(&pf->pdev->dev,
7247                         "set phy config ret =  %s last_status =  %s\n",
7248                         i40e_stat_str(&pf->hw, err),
7249                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7250                 return err;
7251         }
7252
7253         /* Update the link info */
7254         err = i40e_update_link_info(hw);
7255         if (err) {
7256                 /* Wait a little bit (on 40G cards it sometimes takes a really
7257                  * long time for link to come back from the atomic reset)
7258                  * and try once more
7259                  */
7260                 msleep(1000);
7261                 i40e_update_link_info(hw);
7262         }
7263
7264         i40e_aq_set_link_restart_an(hw, is_up, NULL);
7265
7266         return I40E_SUCCESS;
7267 }
7268
7269 /**
7270  * i40e_up - Bring the connection back up after being down
7271  * @vsi: the VSI being configured
7272  **/
7273 int i40e_up(struct i40e_vsi *vsi)
7274 {
7275         int err;
7276
7277         if (vsi->type == I40E_VSI_MAIN &&
7278             (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7279              vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7280                 i40e_force_link_state(vsi->back, true);
7281
7282         err = i40e_vsi_configure(vsi);
7283         if (!err)
7284                 err = i40e_up_complete(vsi);
7285
7286         return err;
7287 }
7288
7289 /**
7290  * i40e_down - Shutdown the connection processing
7291  * @vsi: the VSI being stopped
7292  **/
7293 void i40e_down(struct i40e_vsi *vsi)
7294 {
7295         int i;
7296
7297         /* It is assumed that the caller of this function
7298          * sets the vsi->state __I40E_VSI_DOWN bit.
7299          */
7300         if (vsi->netdev) {
7301                 netif_carrier_off(vsi->netdev);
7302                 netif_tx_disable(vsi->netdev);
7303         }
7304         i40e_vsi_disable_irq(vsi);
7305         i40e_vsi_stop_rings(vsi);
7306         if (vsi->type == I40E_VSI_MAIN &&
7307            (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7308             vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7309                 i40e_force_link_state(vsi->back, false);
7310         i40e_napi_disable_all(vsi);
7311
7312         for (i = 0; i < vsi->num_queue_pairs; i++) {
7313                 i40e_clean_tx_ring(vsi->tx_rings[i]);
7314                 if (i40e_enabled_xdp_vsi(vsi)) {
7315                         /* Make sure that in-progress ndo_xdp_xmit and
7316                          * ndo_xsk_wakeup calls are completed.
7317                          */
7318                         synchronize_rcu();
7319                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
7320                 }
7321                 i40e_clean_rx_ring(vsi->rx_rings[i]);
7322         }
7323
7324 }
7325
7326 /**
7327  * i40e_validate_mqprio_qopt- validate queue mapping info
7328  * @vsi: the VSI being configured
7329  * @mqprio_qopt: queue parametrs
7330  **/
7331 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7332                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
7333 {
7334         u64 sum_max_rate = 0;
7335         u64 max_rate = 0;
7336         int i;
7337
7338         if (mqprio_qopt->qopt.offset[0] != 0 ||
7339             mqprio_qopt->qopt.num_tc < 1 ||
7340             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7341                 return -EINVAL;
7342         for (i = 0; ; i++) {
7343                 if (!mqprio_qopt->qopt.count[i])
7344                         return -EINVAL;
7345                 if (mqprio_qopt->min_rate[i]) {
7346                         dev_err(&vsi->back->pdev->dev,
7347                                 "Invalid min tx rate (greater than 0) specified\n");
7348                         return -EINVAL;
7349                 }
7350                 max_rate = mqprio_qopt->max_rate[i];
7351                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7352                 sum_max_rate += max_rate;
7353
7354                 if (i >= mqprio_qopt->qopt.num_tc - 1)
7355                         break;
7356                 if (mqprio_qopt->qopt.offset[i + 1] !=
7357                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7358                         return -EINVAL;
7359         }
7360         if (vsi->num_queue_pairs <
7361             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7362                 dev_err(&vsi->back->pdev->dev,
7363                         "Failed to create traffic channel, insufficient number of queues.\n");
7364                 return -EINVAL;
7365         }
7366         if (sum_max_rate > i40e_get_link_speed(vsi)) {
7367                 dev_err(&vsi->back->pdev->dev,
7368                         "Invalid max tx rate specified\n");
7369                 return -EINVAL;
7370         }
7371         return 0;
7372 }
7373
7374 /**
7375  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7376  * @vsi: the VSI being configured
7377  **/
7378 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7379 {
7380         u16 qcount;
7381         int i;
7382
7383         /* Only TC0 is enabled */
7384         vsi->tc_config.numtc = 1;
7385         vsi->tc_config.enabled_tc = 1;
7386         qcount = min_t(int, vsi->alloc_queue_pairs,
7387                        i40e_pf_get_max_q_per_tc(vsi->back));
7388         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7389                 /* For the TC that is not enabled set the offset to default
7390                  * queue and allocate one queue for the given TC.
7391                  */
7392                 vsi->tc_config.tc_info[i].qoffset = 0;
7393                 if (i == 0)
7394                         vsi->tc_config.tc_info[i].qcount = qcount;
7395                 else
7396                         vsi->tc_config.tc_info[i].qcount = 1;
7397                 vsi->tc_config.tc_info[i].netdev_tc = 0;
7398         }
7399 }
7400
7401 /**
7402  * i40e_del_macvlan_filter
7403  * @hw: pointer to the HW structure
7404  * @seid: seid of the channel VSI
7405  * @macaddr: the mac address to apply as a filter
7406  * @aq_err: store the admin Q error
7407  *
7408  * This function deletes a mac filter on the channel VSI which serves as the
7409  * macvlan. Returns 0 on success.
7410  **/
7411 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7412                                            const u8 *macaddr, int *aq_err)
7413 {
7414         struct i40e_aqc_remove_macvlan_element_data element;
7415         i40e_status status;
7416
7417         memset(&element, 0, sizeof(element));
7418         ether_addr_copy(element.mac_addr, macaddr);
7419         element.vlan_tag = 0;
7420         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7421         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7422         *aq_err = hw->aq.asq_last_status;
7423
7424         return status;
7425 }
7426
7427 /**
7428  * i40e_add_macvlan_filter
7429  * @hw: pointer to the HW structure
7430  * @seid: seid of the channel VSI
7431  * @macaddr: the mac address to apply as a filter
7432  * @aq_err: store the admin Q error
7433  *
7434  * This function adds a mac filter on the channel VSI which serves as the
7435  * macvlan. Returns 0 on success.
7436  **/
7437 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7438                                            const u8 *macaddr, int *aq_err)
7439 {
7440         struct i40e_aqc_add_macvlan_element_data element;
7441         i40e_status status;
7442         u16 cmd_flags = 0;
7443
7444         ether_addr_copy(element.mac_addr, macaddr);
7445         element.vlan_tag = 0;
7446         element.queue_number = 0;
7447         element.match_method = I40E_AQC_MM_ERR_NO_RES;
7448         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7449         element.flags = cpu_to_le16(cmd_flags);
7450         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7451         *aq_err = hw->aq.asq_last_status;
7452
7453         return status;
7454 }
7455
7456 /**
7457  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7458  * @vsi: the VSI we want to access
7459  * @ch: the channel we want to access
7460  */
7461 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7462 {
7463         struct i40e_ring *tx_ring, *rx_ring;
7464         u16 pf_q;
7465         int i;
7466
7467         for (i = 0; i < ch->num_queue_pairs; i++) {
7468                 pf_q = ch->base_queue + i;
7469                 tx_ring = vsi->tx_rings[pf_q];
7470                 tx_ring->ch = NULL;
7471                 rx_ring = vsi->rx_rings[pf_q];
7472                 rx_ring->ch = NULL;
7473         }
7474 }
7475
7476 /**
7477  * i40e_free_macvlan_channels
7478  * @vsi: the VSI we want to access
7479  *
7480  * This function frees the Qs of the channel VSI from
7481  * the stack and also deletes the channel VSIs which
7482  * serve as macvlans.
7483  */
7484 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7485 {
7486         struct i40e_channel *ch, *ch_tmp;
7487         int ret;
7488
7489         if (list_empty(&vsi->macvlan_list))
7490                 return;
7491
7492         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7493                 struct i40e_vsi *parent_vsi;
7494
7495                 if (i40e_is_channel_macvlan(ch)) {
7496                         i40e_reset_ch_rings(vsi, ch);
7497                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7498                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7499                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7500                         kfree(ch->fwd);
7501                         ch->fwd = NULL;
7502                 }
7503
7504                 list_del(&ch->list);
7505                 parent_vsi = ch->parent_vsi;
7506                 if (!parent_vsi || !ch->initialized) {
7507                         kfree(ch);
7508                         continue;
7509                 }
7510
7511                 /* remove the VSI */
7512                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7513                                              NULL);
7514                 if (ret)
7515                         dev_err(&vsi->back->pdev->dev,
7516                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7517                                 ch->seid, parent_vsi->seid);
7518                 kfree(ch);
7519         }
7520         vsi->macvlan_cnt = 0;
7521 }
7522
7523 /**
7524  * i40e_fwd_ring_up - bring the macvlan device up
7525  * @vsi: the VSI we want to access
7526  * @vdev: macvlan netdevice
7527  * @fwd: the private fwd structure
7528  */
7529 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7530                             struct i40e_fwd_adapter *fwd)
7531 {
7532         int ret = 0, num_tc = 1,  i, aq_err;
7533         struct i40e_channel *ch, *ch_tmp;
7534         struct i40e_pf *pf = vsi->back;
7535         struct i40e_hw *hw = &pf->hw;
7536
7537         if (list_empty(&vsi->macvlan_list))
7538                 return -EINVAL;
7539
7540         /* Go through the list and find an available channel */
7541         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7542                 if (!i40e_is_channel_macvlan(ch)) {
7543                         ch->fwd = fwd;
7544                         /* record configuration for macvlan interface in vdev */
7545                         for (i = 0; i < num_tc; i++)
7546                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7547                                                              i,
7548                                                              ch->num_queue_pairs,
7549                                                              ch->base_queue);
7550                         for (i = 0; i < ch->num_queue_pairs; i++) {
7551                                 struct i40e_ring *tx_ring, *rx_ring;
7552                                 u16 pf_q;
7553
7554                                 pf_q = ch->base_queue + i;
7555
7556                                 /* Get to TX ring ptr */
7557                                 tx_ring = vsi->tx_rings[pf_q];
7558                                 tx_ring->ch = ch;
7559
7560                                 /* Get the RX ring ptr */
7561                                 rx_ring = vsi->rx_rings[pf_q];
7562                                 rx_ring->ch = ch;
7563                         }
7564                         break;
7565                 }
7566         }
7567
7568         /* Guarantee all rings are updated before we update the
7569          * MAC address filter.
7570          */
7571         wmb();
7572
7573         /* Add a mac filter */
7574         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7575         if (ret) {
7576                 /* if we cannot add the MAC rule then disable the offload */
7577                 macvlan_release_l2fw_offload(vdev);
7578                 for (i = 0; i < ch->num_queue_pairs; i++) {
7579                         struct i40e_ring *rx_ring;
7580                         u16 pf_q;
7581
7582                         pf_q = ch->base_queue + i;
7583                         rx_ring = vsi->rx_rings[pf_q];
7584                         rx_ring->netdev = NULL;
7585                 }
7586                 dev_info(&pf->pdev->dev,
7587                          "Error adding mac filter on macvlan err %s, aq_err %s\n",
7588                           i40e_stat_str(hw, ret),
7589                           i40e_aq_str(hw, aq_err));
7590                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7591         }
7592
7593         return ret;
7594 }
7595
7596 /**
7597  * i40e_setup_macvlans - create the channels which will be macvlans
7598  * @vsi: the VSI we want to access
7599  * @macvlan_cnt: no. of macvlans to be setup
7600  * @qcnt: no. of Qs per macvlan
7601  * @vdev: macvlan netdevice
7602  */
7603 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7604                                struct net_device *vdev)
7605 {
7606         struct i40e_pf *pf = vsi->back;
7607         struct i40e_hw *hw = &pf->hw;
7608         struct i40e_vsi_context ctxt;
7609         u16 sections, qmap, num_qps;
7610         struct i40e_channel *ch;
7611         int i, pow, ret = 0;
7612         u8 offset = 0;
7613
7614         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7615                 return -EINVAL;
7616
7617         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7618
7619         /* find the next higher power-of-2 of num queue pairs */
7620         pow = fls(roundup_pow_of_two(num_qps) - 1);
7621
7622         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7623                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7624
7625         /* Setup context bits for the main VSI */
7626         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7627         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7628         memset(&ctxt, 0, sizeof(ctxt));
7629         ctxt.seid = vsi->seid;
7630         ctxt.pf_num = vsi->back->hw.pf_id;
7631         ctxt.vf_num = 0;
7632         ctxt.uplink_seid = vsi->uplink_seid;
7633         ctxt.info = vsi->info;
7634         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7635         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7636         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7637         ctxt.info.valid_sections |= cpu_to_le16(sections);
7638
7639         /* Reconfigure RSS for main VSI with new max queue count */
7640         vsi->rss_size = max_t(u16, num_qps, qcnt);
7641         ret = i40e_vsi_config_rss(vsi);
7642         if (ret) {
7643                 dev_info(&pf->pdev->dev,
7644                          "Failed to reconfig RSS for num_queues (%u)\n",
7645                          vsi->rss_size);
7646                 return ret;
7647         }
7648         vsi->reconfig_rss = true;
7649         dev_dbg(&vsi->back->pdev->dev,
7650                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7651         vsi->next_base_queue = num_qps;
7652         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7653
7654         /* Update the VSI after updating the VSI queue-mapping
7655          * information
7656          */
7657         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7658         if (ret) {
7659                 dev_info(&pf->pdev->dev,
7660                          "Update vsi tc config failed, err %s aq_err %s\n",
7661                          i40e_stat_str(hw, ret),
7662                          i40e_aq_str(hw, hw->aq.asq_last_status));
7663                 return ret;
7664         }
7665         /* update the local VSI info with updated queue map */
7666         i40e_vsi_update_queue_map(vsi, &ctxt);
7667         vsi->info.valid_sections = 0;
7668
7669         /* Create channels for macvlans */
7670         INIT_LIST_HEAD(&vsi->macvlan_list);
7671         for (i = 0; i < macvlan_cnt; i++) {
7672                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7673                 if (!ch) {
7674                         ret = -ENOMEM;
7675                         goto err_free;
7676                 }
7677                 INIT_LIST_HEAD(&ch->list);
7678                 ch->num_queue_pairs = qcnt;
7679                 if (!i40e_setup_channel(pf, vsi, ch)) {
7680                         ret = -EINVAL;
7681                         kfree(ch);
7682                         goto err_free;
7683                 }
7684                 ch->parent_vsi = vsi;
7685                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7686                 vsi->macvlan_cnt++;
7687                 list_add_tail(&ch->list, &vsi->macvlan_list);
7688         }
7689
7690         return ret;
7691
7692 err_free:
7693         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7694         i40e_free_macvlan_channels(vsi);
7695
7696         return ret;
7697 }
7698
7699 /**
7700  * i40e_fwd_add - configure macvlans
7701  * @netdev: net device to configure
7702  * @vdev: macvlan netdevice
7703  **/
7704 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7705 {
7706         struct i40e_netdev_priv *np = netdev_priv(netdev);
7707         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7708         struct i40e_vsi *vsi = np->vsi;
7709         struct i40e_pf *pf = vsi->back;
7710         struct i40e_fwd_adapter *fwd;
7711         int avail_macvlan, ret;
7712
7713         if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7714                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7715                 return ERR_PTR(-EINVAL);
7716         }
7717         if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7718                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7719                 return ERR_PTR(-EINVAL);
7720         }
7721         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7722                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7723                 return ERR_PTR(-EINVAL);
7724         }
7725
7726         /* The macvlan device has to be a single Q device so that the
7727          * tc_to_txq field can be reused to pick the tx queue.
7728          */
7729         if (netif_is_multiqueue(vdev))
7730                 return ERR_PTR(-ERANGE);
7731
7732         if (!vsi->macvlan_cnt) {
7733                 /* reserve bit 0 for the pf device */
7734                 set_bit(0, vsi->fwd_bitmask);
7735
7736                 /* Try to reserve as many queues as possible for macvlans. First
7737                  * reserve 3/4th of max vectors, then half, then quarter and
7738                  * calculate Qs per macvlan as you go
7739                  */
7740                 vectors = pf->num_lan_msix;
7741                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7742                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7743                         q_per_macvlan = 4;
7744                         macvlan_cnt = (vectors - 32) / 4;
7745                 } else if (vectors <= 64 && vectors > 32) {
7746                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7747                         q_per_macvlan = 2;
7748                         macvlan_cnt = (vectors - 16) / 2;
7749                 } else if (vectors <= 32 && vectors > 16) {
7750                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7751                         q_per_macvlan = 1;
7752                         macvlan_cnt = vectors - 16;
7753                 } else if (vectors <= 16 && vectors > 8) {
7754                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
7755                         q_per_macvlan = 1;
7756                         macvlan_cnt = vectors - 8;
7757                 } else {
7758                         /* allocate 1 Q per macvlan and 1 Q to the PF */
7759                         q_per_macvlan = 1;
7760                         macvlan_cnt = vectors - 1;
7761                 }
7762
7763                 if (macvlan_cnt == 0)
7764                         return ERR_PTR(-EBUSY);
7765
7766                 /* Quiesce VSI queues */
7767                 i40e_quiesce_vsi(vsi);
7768
7769                 /* sets up the macvlans but does not "enable" them */
7770                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7771                                           vdev);
7772                 if (ret)
7773                         return ERR_PTR(ret);
7774
7775                 /* Unquiesce VSI */
7776                 i40e_unquiesce_vsi(vsi);
7777         }
7778         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7779                                             vsi->macvlan_cnt);
7780         if (avail_macvlan >= I40E_MAX_MACVLANS)
7781                 return ERR_PTR(-EBUSY);
7782
7783         /* create the fwd struct */
7784         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7785         if (!fwd)
7786                 return ERR_PTR(-ENOMEM);
7787
7788         set_bit(avail_macvlan, vsi->fwd_bitmask);
7789         fwd->bit_no = avail_macvlan;
7790         netdev_set_sb_channel(vdev, avail_macvlan);
7791         fwd->netdev = vdev;
7792
7793         if (!netif_running(netdev))
7794                 return fwd;
7795
7796         /* Set fwd ring up */
7797         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7798         if (ret) {
7799                 /* unbind the queues and drop the subordinate channel config */
7800                 netdev_unbind_sb_channel(netdev, vdev);
7801                 netdev_set_sb_channel(vdev, 0);
7802
7803                 kfree(fwd);
7804                 return ERR_PTR(-EINVAL);
7805         }
7806
7807         return fwd;
7808 }
7809
7810 /**
7811  * i40e_del_all_macvlans - Delete all the mac filters on the channels
7812  * @vsi: the VSI we want to access
7813  */
7814 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7815 {
7816         struct i40e_channel *ch, *ch_tmp;
7817         struct i40e_pf *pf = vsi->back;
7818         struct i40e_hw *hw = &pf->hw;
7819         int aq_err, ret = 0;
7820
7821         if (list_empty(&vsi->macvlan_list))
7822                 return;
7823
7824         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7825                 if (i40e_is_channel_macvlan(ch)) {
7826                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7827                                                       i40e_channel_mac(ch),
7828                                                       &aq_err);
7829                         if (!ret) {
7830                                 /* Reset queue contexts */
7831                                 i40e_reset_ch_rings(vsi, ch);
7832                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7833                                 netdev_unbind_sb_channel(vsi->netdev,
7834                                                          ch->fwd->netdev);
7835                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
7836                                 kfree(ch->fwd);
7837                                 ch->fwd = NULL;
7838                         }
7839                 }
7840         }
7841 }
7842
7843 /**
7844  * i40e_fwd_del - delete macvlan interfaces
7845  * @netdev: net device to configure
7846  * @vdev: macvlan netdevice
7847  */
7848 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7849 {
7850         struct i40e_netdev_priv *np = netdev_priv(netdev);
7851         struct i40e_fwd_adapter *fwd = vdev;
7852         struct i40e_channel *ch, *ch_tmp;
7853         struct i40e_vsi *vsi = np->vsi;
7854         struct i40e_pf *pf = vsi->back;
7855         struct i40e_hw *hw = &pf->hw;
7856         int aq_err, ret = 0;
7857
7858         /* Find the channel associated with the macvlan and del mac filter */
7859         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7860                 if (i40e_is_channel_macvlan(ch) &&
7861                     ether_addr_equal(i40e_channel_mac(ch),
7862                                      fwd->netdev->dev_addr)) {
7863                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7864                                                       i40e_channel_mac(ch),
7865                                                       &aq_err);
7866                         if (!ret) {
7867                                 /* Reset queue contexts */
7868                                 i40e_reset_ch_rings(vsi, ch);
7869                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7870                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
7871                                 netdev_set_sb_channel(fwd->netdev, 0);
7872                                 kfree(ch->fwd);
7873                                 ch->fwd = NULL;
7874                         } else {
7875                                 dev_info(&pf->pdev->dev,
7876                                          "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7877                                           i40e_stat_str(hw, ret),
7878                                           i40e_aq_str(hw, aq_err));
7879                         }
7880                         break;
7881                 }
7882         }
7883 }
7884
7885 /**
7886  * i40e_setup_tc - configure multiple traffic classes
7887  * @netdev: net device to configure
7888  * @type_data: tc offload data
7889  **/
7890 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7891 {
7892         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7893         struct i40e_netdev_priv *np = netdev_priv(netdev);
7894         struct i40e_vsi *vsi = np->vsi;
7895         struct i40e_pf *pf = vsi->back;
7896         u8 enabled_tc = 0, num_tc, hw;
7897         bool need_reset = false;
7898         int old_queue_pairs;
7899         int ret = -EINVAL;
7900         u16 mode;
7901         int i;
7902
7903         old_queue_pairs = vsi->num_queue_pairs;
7904         num_tc = mqprio_qopt->qopt.num_tc;
7905         hw = mqprio_qopt->qopt.hw;
7906         mode = mqprio_qopt->mode;
7907         if (!hw) {
7908                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7909                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7910                 goto config_tc;
7911         }
7912
7913         /* Check if MFP enabled */
7914         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7915                 netdev_info(netdev,
7916                             "Configuring TC not supported in MFP mode\n");
7917                 return ret;
7918         }
7919         switch (mode) {
7920         case TC_MQPRIO_MODE_DCB:
7921                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7922
7923                 /* Check if DCB enabled to continue */
7924                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7925                         netdev_info(netdev,
7926                                     "DCB is not enabled for adapter\n");
7927                         return ret;
7928                 }
7929
7930                 /* Check whether tc count is within enabled limit */
7931                 if (num_tc > i40e_pf_get_num_tc(pf)) {
7932                         netdev_info(netdev,
7933                                     "TC count greater than enabled on link for adapter\n");
7934                         return ret;
7935                 }
7936                 break;
7937         case TC_MQPRIO_MODE_CHANNEL:
7938                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7939                         netdev_info(netdev,
7940                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7941                         return ret;
7942                 }
7943                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7944                         return ret;
7945                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7946                 if (ret)
7947                         return ret;
7948                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7949                        sizeof(*mqprio_qopt));
7950                 pf->flags |= I40E_FLAG_TC_MQPRIO;
7951                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7952                 break;
7953         default:
7954                 return -EINVAL;
7955         }
7956
7957 config_tc:
7958         /* Generate TC map for number of tc requested */
7959         for (i = 0; i < num_tc; i++)
7960                 enabled_tc |= BIT(i);
7961
7962         /* Requesting same TC configuration as already enabled */
7963         if (enabled_tc == vsi->tc_config.enabled_tc &&
7964             mode != TC_MQPRIO_MODE_CHANNEL)
7965                 return 0;
7966
7967         /* Quiesce VSI queues */
7968         i40e_quiesce_vsi(vsi);
7969
7970         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7971                 i40e_remove_queue_channels(vsi);
7972
7973         /* Configure VSI for enabled TCs */
7974         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7975         if (ret) {
7976                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7977                             vsi->seid);
7978                 need_reset = true;
7979                 goto exit;
7980         } else if (enabled_tc &&
7981                    (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
7982                 netdev_info(netdev,
7983                             "Failed to create channel. Override queues (%u) not power of 2\n",
7984                             vsi->tc_config.tc_info[0].qcount);
7985                 ret = -EINVAL;
7986                 need_reset = true;
7987                 goto exit;
7988         }
7989
7990         dev_info(&vsi->back->pdev->dev,
7991                  "Setup channel (id:%u) utilizing num_queues %d\n",
7992                  vsi->seid, vsi->tc_config.tc_info[0].qcount);
7993
7994         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7995                 if (vsi->mqprio_qopt.max_rate[0]) {
7996                         u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
7997
7998                         do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
7999                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8000                         if (!ret) {
8001                                 u64 credits = max_tx_rate;
8002
8003                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
8004                                 dev_dbg(&vsi->back->pdev->dev,
8005                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8006                                         max_tx_rate,
8007                                         credits,
8008                                         vsi->seid);
8009                         } else {
8010                                 need_reset = true;
8011                                 goto exit;
8012                         }
8013                 }
8014                 ret = i40e_configure_queue_channels(vsi);
8015                 if (ret) {
8016                         vsi->num_queue_pairs = old_queue_pairs;
8017                         netdev_info(netdev,
8018                                     "Failed configuring queue channels\n");
8019                         need_reset = true;
8020                         goto exit;
8021                 }
8022         }
8023
8024 exit:
8025         /* Reset the configuration data to defaults, only TC0 is enabled */
8026         if (need_reset) {
8027                 i40e_vsi_set_default_tc_config(vsi);
8028                 need_reset = false;
8029         }
8030
8031         /* Unquiesce VSI */
8032         i40e_unquiesce_vsi(vsi);
8033         return ret;
8034 }
8035
8036 /**
8037  * i40e_set_cld_element - sets cloud filter element data
8038  * @filter: cloud filter rule
8039  * @cld: ptr to cloud filter element data
8040  *
8041  * This is helper function to copy data into cloud filter element
8042  **/
8043 static inline void
8044 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8045                      struct i40e_aqc_cloud_filters_element_data *cld)
8046 {
8047         u32 ipa;
8048         int i;
8049
8050         memset(cld, 0, sizeof(*cld));
8051         ether_addr_copy(cld->outer_mac, filter->dst_mac);
8052         ether_addr_copy(cld->inner_mac, filter->src_mac);
8053
8054         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8055                 return;
8056
8057         if (filter->n_proto == ETH_P_IPV6) {
8058 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
8059                 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8060                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8061
8062                         *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8063                 }
8064         } else {
8065                 ipa = be32_to_cpu(filter->dst_ipv4);
8066
8067                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8068         }
8069
8070         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8071
8072         /* tenant_id is not supported by FW now, once the support is enabled
8073          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8074          */
8075         if (filter->tenant_id)
8076                 return;
8077 }
8078
8079 /**
8080  * i40e_add_del_cloud_filter - Add/del cloud filter
8081  * @vsi: pointer to VSI
8082  * @filter: cloud filter rule
8083  * @add: if true, add, if false, delete
8084  *
8085  * Add or delete a cloud filter for a specific flow spec.
8086  * Returns 0 if the filter were successfully added.
8087  **/
8088 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8089                               struct i40e_cloud_filter *filter, bool add)
8090 {
8091         struct i40e_aqc_cloud_filters_element_data cld_filter;
8092         struct i40e_pf *pf = vsi->back;
8093         int ret;
8094         static const u16 flag_table[128] = {
8095                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8096                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8097                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8098                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8099                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8100                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8101                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8102                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8103                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8104                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8105                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8106                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8107                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8108                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
8109         };
8110
8111         if (filter->flags >= ARRAY_SIZE(flag_table))
8112                 return I40E_ERR_CONFIG;
8113
8114         memset(&cld_filter, 0, sizeof(cld_filter));
8115
8116         /* copy element needed to add cloud filter from filter */
8117         i40e_set_cld_element(filter, &cld_filter);
8118
8119         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8120                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8121                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8122
8123         if (filter->n_proto == ETH_P_IPV6)
8124                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8125                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8126         else
8127                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8128                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8129
8130         if (add)
8131                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8132                                                 &cld_filter, 1);
8133         else
8134                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8135                                                 &cld_filter, 1);
8136         if (ret)
8137                 dev_dbg(&pf->pdev->dev,
8138                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8139                         add ? "add" : "delete", filter->dst_port, ret,
8140                         pf->hw.aq.asq_last_status);
8141         else
8142                 dev_info(&pf->pdev->dev,
8143                          "%s cloud filter for VSI: %d\n",
8144                          add ? "Added" : "Deleted", filter->seid);
8145         return ret;
8146 }
8147
8148 /**
8149  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8150  * @vsi: pointer to VSI
8151  * @filter: cloud filter rule
8152  * @add: if true, add, if false, delete
8153  *
8154  * Add or delete a cloud filter for a specific flow spec using big buffer.
8155  * Returns 0 if the filter were successfully added.
8156  **/
8157 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8158                                       struct i40e_cloud_filter *filter,
8159                                       bool add)
8160 {
8161         struct i40e_aqc_cloud_filters_element_bb cld_filter;
8162         struct i40e_pf *pf = vsi->back;
8163         int ret;
8164
8165         /* Both (src/dst) valid mac_addr are not supported */
8166         if ((is_valid_ether_addr(filter->dst_mac) &&
8167              is_valid_ether_addr(filter->src_mac)) ||
8168             (is_multicast_ether_addr(filter->dst_mac) &&
8169              is_multicast_ether_addr(filter->src_mac)))
8170                 return -EOPNOTSUPP;
8171
8172         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8173          * ports are not supported via big buffer now.
8174          */
8175         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8176                 return -EOPNOTSUPP;
8177
8178         /* adding filter using src_port/src_ip is not supported at this stage */
8179         if (filter->src_port ||
8180             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8181             !ipv6_addr_any(&filter->ip.v6.src_ip6))
8182                 return -EOPNOTSUPP;
8183
8184         memset(&cld_filter, 0, sizeof(cld_filter));
8185
8186         /* copy element needed to add cloud filter from filter */
8187         i40e_set_cld_element(filter, &cld_filter.element);
8188
8189         if (is_valid_ether_addr(filter->dst_mac) ||
8190             is_valid_ether_addr(filter->src_mac) ||
8191             is_multicast_ether_addr(filter->dst_mac) ||
8192             is_multicast_ether_addr(filter->src_mac)) {
8193                 /* MAC + IP : unsupported mode */
8194                 if (filter->dst_ipv4)
8195                         return -EOPNOTSUPP;
8196
8197                 /* since we validated that L4 port must be valid before
8198                  * we get here, start with respective "flags" value
8199                  * and update if vlan is present or not
8200                  */
8201                 cld_filter.element.flags =
8202                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8203
8204                 if (filter->vlan_id) {
8205                         cld_filter.element.flags =
8206                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8207                 }
8208
8209         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8210                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8211                 cld_filter.element.flags =
8212                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8213                 if (filter->n_proto == ETH_P_IPV6)
8214                         cld_filter.element.flags |=
8215                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8216                 else
8217                         cld_filter.element.flags |=
8218                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8219         } else {
8220                 dev_err(&pf->pdev->dev,
8221                         "either mac or ip has to be valid for cloud filter\n");
8222                 return -EINVAL;
8223         }
8224
8225         /* Now copy L4 port in Byte 6..7 in general fields */
8226         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8227                                                 be16_to_cpu(filter->dst_port);
8228
8229         if (add) {
8230                 /* Validate current device switch mode, change if necessary */
8231                 ret = i40e_validate_and_set_switch_mode(vsi);
8232                 if (ret) {
8233                         dev_err(&pf->pdev->dev,
8234                                 "failed to set switch mode, ret %d\n",
8235                                 ret);
8236                         return ret;
8237                 }
8238
8239                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8240                                                    &cld_filter, 1);
8241         } else {
8242                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8243                                                    &cld_filter, 1);
8244         }
8245
8246         if (ret)
8247                 dev_dbg(&pf->pdev->dev,
8248                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8249                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8250         else
8251                 dev_info(&pf->pdev->dev,
8252                          "%s cloud filter for VSI: %d, L4 port: %d\n",
8253                          add ? "add" : "delete", filter->seid,
8254                          ntohs(filter->dst_port));
8255         return ret;
8256 }
8257
8258 /**
8259  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8260  * @vsi: Pointer to VSI
8261  * @f: Pointer to struct flow_cls_offload
8262  * @filter: Pointer to cloud filter structure
8263  *
8264  **/
8265 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8266                                  struct flow_cls_offload *f,
8267                                  struct i40e_cloud_filter *filter)
8268 {
8269         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8270         struct flow_dissector *dissector = rule->match.dissector;
8271         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8272         struct i40e_pf *pf = vsi->back;
8273         u8 field_flags = 0;
8274
8275         if (dissector->used_keys &
8276             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8277               BIT(FLOW_DISSECTOR_KEY_BASIC) |
8278               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8279               BIT(FLOW_DISSECTOR_KEY_VLAN) |
8280               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8281               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8282               BIT(FLOW_DISSECTOR_KEY_PORTS) |
8283               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8284                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8285                         dissector->used_keys);
8286                 return -EOPNOTSUPP;
8287         }
8288
8289         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8290                 struct flow_match_enc_keyid match;
8291
8292                 flow_rule_match_enc_keyid(rule, &match);
8293                 if (match.mask->keyid != 0)
8294                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8295
8296                 filter->tenant_id = be32_to_cpu(match.key->keyid);
8297         }
8298
8299         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8300                 struct flow_match_basic match;
8301
8302                 flow_rule_match_basic(rule, &match);
8303                 n_proto_key = ntohs(match.key->n_proto);
8304                 n_proto_mask = ntohs(match.mask->n_proto);
8305
8306                 if (n_proto_key == ETH_P_ALL) {
8307                         n_proto_key = 0;
8308                         n_proto_mask = 0;
8309                 }
8310                 filter->n_proto = n_proto_key & n_proto_mask;
8311                 filter->ip_proto = match.key->ip_proto;
8312         }
8313
8314         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8315                 struct flow_match_eth_addrs match;
8316
8317                 flow_rule_match_eth_addrs(rule, &match);
8318
8319                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8320                 if (!is_zero_ether_addr(match.mask->dst)) {
8321                         if (is_broadcast_ether_addr(match.mask->dst)) {
8322                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
8323                         } else {
8324                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8325                                         match.mask->dst);
8326                                 return I40E_ERR_CONFIG;
8327                         }
8328                 }
8329
8330                 if (!is_zero_ether_addr(match.mask->src)) {
8331                         if (is_broadcast_ether_addr(match.mask->src)) {
8332                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
8333                         } else {
8334                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8335                                         match.mask->src);
8336                                 return I40E_ERR_CONFIG;
8337                         }
8338                 }
8339                 ether_addr_copy(filter->dst_mac, match.key->dst);
8340                 ether_addr_copy(filter->src_mac, match.key->src);
8341         }
8342
8343         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8344                 struct flow_match_vlan match;
8345
8346                 flow_rule_match_vlan(rule, &match);
8347                 if (match.mask->vlan_id) {
8348                         if (match.mask->vlan_id == VLAN_VID_MASK) {
8349                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8350
8351                         } else {
8352                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8353                                         match.mask->vlan_id);
8354                                 return I40E_ERR_CONFIG;
8355                         }
8356                 }
8357
8358                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8359         }
8360
8361         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8362                 struct flow_match_control match;
8363
8364                 flow_rule_match_control(rule, &match);
8365                 addr_type = match.key->addr_type;
8366         }
8367
8368         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8369                 struct flow_match_ipv4_addrs match;
8370
8371                 flow_rule_match_ipv4_addrs(rule, &match);
8372                 if (match.mask->dst) {
8373                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8374                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8375                         } else {
8376                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8377                                         &match.mask->dst);
8378                                 return I40E_ERR_CONFIG;
8379                         }
8380                 }
8381
8382                 if (match.mask->src) {
8383                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
8384                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8385                         } else {
8386                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8387                                         &match.mask->src);
8388                                 return I40E_ERR_CONFIG;
8389                         }
8390                 }
8391
8392                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8393                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8394                         return I40E_ERR_CONFIG;
8395                 }
8396                 filter->dst_ipv4 = match.key->dst;
8397                 filter->src_ipv4 = match.key->src;
8398         }
8399
8400         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8401                 struct flow_match_ipv6_addrs match;
8402
8403                 flow_rule_match_ipv6_addrs(rule, &match);
8404
8405                 /* src and dest IPV6 address should not be LOOPBACK
8406                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
8407                  */
8408                 if (ipv6_addr_loopback(&match.key->dst) ||
8409                     ipv6_addr_loopback(&match.key->src)) {
8410                         dev_err(&pf->pdev->dev,
8411                                 "Bad ipv6, addr is LOOPBACK\n");
8412                         return I40E_ERR_CONFIG;
8413                 }
8414                 if (!ipv6_addr_any(&match.mask->dst) ||
8415                     !ipv6_addr_any(&match.mask->src))
8416                         field_flags |= I40E_CLOUD_FIELD_IIP;
8417
8418                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8419                        sizeof(filter->src_ipv6));
8420                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8421                        sizeof(filter->dst_ipv6));
8422         }
8423
8424         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8425                 struct flow_match_ports match;
8426
8427                 flow_rule_match_ports(rule, &match);
8428                 if (match.mask->src) {
8429                         if (match.mask->src == cpu_to_be16(0xffff)) {
8430                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8431                         } else {
8432                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8433                                         be16_to_cpu(match.mask->src));
8434                                 return I40E_ERR_CONFIG;
8435                         }
8436                 }
8437
8438                 if (match.mask->dst) {
8439                         if (match.mask->dst == cpu_to_be16(0xffff)) {
8440                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8441                         } else {
8442                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8443                                         be16_to_cpu(match.mask->dst));
8444                                 return I40E_ERR_CONFIG;
8445                         }
8446                 }
8447
8448                 filter->dst_port = match.key->dst;
8449                 filter->src_port = match.key->src;
8450
8451                 switch (filter->ip_proto) {
8452                 case IPPROTO_TCP:
8453                 case IPPROTO_UDP:
8454                         break;
8455                 default:
8456                         dev_err(&pf->pdev->dev,
8457                                 "Only UDP and TCP transport are supported\n");
8458                         return -EINVAL;
8459                 }
8460         }
8461         filter->flags = field_flags;
8462         return 0;
8463 }
8464
8465 /**
8466  * i40e_handle_tclass: Forward to a traffic class on the device
8467  * @vsi: Pointer to VSI
8468  * @tc: traffic class index on the device
8469  * @filter: Pointer to cloud filter structure
8470  *
8471  **/
8472 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8473                               struct i40e_cloud_filter *filter)
8474 {
8475         struct i40e_channel *ch, *ch_tmp;
8476
8477         /* direct to a traffic class on the same device */
8478         if (tc == 0) {
8479                 filter->seid = vsi->seid;
8480                 return 0;
8481         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8482                 if (!filter->dst_port) {
8483                         dev_err(&vsi->back->pdev->dev,
8484                                 "Specify destination port to direct to traffic class that is not default\n");
8485                         return -EINVAL;
8486                 }
8487                 if (list_empty(&vsi->ch_list))
8488                         return -EINVAL;
8489                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8490                                          list) {
8491                         if (ch->seid == vsi->tc_seid_map[tc])
8492                                 filter->seid = ch->seid;
8493                 }
8494                 return 0;
8495         }
8496         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8497         return -EINVAL;
8498 }
8499
8500 /**
8501  * i40e_configure_clsflower - Configure tc flower filters
8502  * @vsi: Pointer to VSI
8503  * @cls_flower: Pointer to struct flow_cls_offload
8504  *
8505  **/
8506 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8507                                     struct flow_cls_offload *cls_flower)
8508 {
8509         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8510         struct i40e_cloud_filter *filter = NULL;
8511         struct i40e_pf *pf = vsi->back;
8512         int err = 0;
8513
8514         if (tc < 0) {
8515                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8516                 return -EOPNOTSUPP;
8517         }
8518
8519         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8520             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8521                 return -EBUSY;
8522
8523         if (pf->fdir_pf_active_filters ||
8524             (!hlist_empty(&pf->fdir_filter_list))) {
8525                 dev_err(&vsi->back->pdev->dev,
8526                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8527                 return -EINVAL;
8528         }
8529
8530         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8531                 dev_err(&vsi->back->pdev->dev,
8532                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8533                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8534                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8535         }
8536
8537         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8538         if (!filter)
8539                 return -ENOMEM;
8540
8541         filter->cookie = cls_flower->cookie;
8542
8543         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8544         if (err < 0)
8545                 goto err;
8546
8547         err = i40e_handle_tclass(vsi, tc, filter);
8548         if (err < 0)
8549                 goto err;
8550
8551         /* Add cloud filter */
8552         if (filter->dst_port)
8553                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8554         else
8555                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8556
8557         if (err) {
8558                 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8559                         err);
8560                 goto err;
8561         }
8562
8563         /* add filter to the ordered list */
8564         INIT_HLIST_NODE(&filter->cloud_node);
8565
8566         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8567
8568         pf->num_cloud_filters++;
8569
8570         return err;
8571 err:
8572         kfree(filter);
8573         return err;
8574 }
8575
8576 /**
8577  * i40e_find_cloud_filter - Find the could filter in the list
8578  * @vsi: Pointer to VSI
8579  * @cookie: filter specific cookie
8580  *
8581  **/
8582 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8583                                                         unsigned long *cookie)
8584 {
8585         struct i40e_cloud_filter *filter = NULL;
8586         struct hlist_node *node2;
8587
8588         hlist_for_each_entry_safe(filter, node2,
8589                                   &vsi->back->cloud_filter_list, cloud_node)
8590                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8591                         return filter;
8592         return NULL;
8593 }
8594
8595 /**
8596  * i40e_delete_clsflower - Remove tc flower filters
8597  * @vsi: Pointer to VSI
8598  * @cls_flower: Pointer to struct flow_cls_offload
8599  *
8600  **/
8601 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8602                                  struct flow_cls_offload *cls_flower)
8603 {
8604         struct i40e_cloud_filter *filter = NULL;
8605         struct i40e_pf *pf = vsi->back;
8606         int err = 0;
8607
8608         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8609
8610         if (!filter)
8611                 return -EINVAL;
8612
8613         hash_del(&filter->cloud_node);
8614
8615         if (filter->dst_port)
8616                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8617         else
8618                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8619
8620         kfree(filter);
8621         if (err) {
8622                 dev_err(&pf->pdev->dev,
8623                         "Failed to delete cloud filter, err %s\n",
8624                         i40e_stat_str(&pf->hw, err));
8625                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8626         }
8627
8628         pf->num_cloud_filters--;
8629         if (!pf->num_cloud_filters)
8630                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8631                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8632                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8633                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8634                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8635                 }
8636         return 0;
8637 }
8638
8639 /**
8640  * i40e_setup_tc_cls_flower - flower classifier offloads
8641  * @np: net device to configure
8642  * @cls_flower: offload data
8643  **/
8644 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8645                                     struct flow_cls_offload *cls_flower)
8646 {
8647         struct i40e_vsi *vsi = np->vsi;
8648
8649         switch (cls_flower->command) {
8650         case FLOW_CLS_REPLACE:
8651                 return i40e_configure_clsflower(vsi, cls_flower);
8652         case FLOW_CLS_DESTROY:
8653                 return i40e_delete_clsflower(vsi, cls_flower);
8654         case FLOW_CLS_STATS:
8655                 return -EOPNOTSUPP;
8656         default:
8657                 return -EOPNOTSUPP;
8658         }
8659 }
8660
8661 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8662                                   void *cb_priv)
8663 {
8664         struct i40e_netdev_priv *np = cb_priv;
8665
8666         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8667                 return -EOPNOTSUPP;
8668
8669         switch (type) {
8670         case TC_SETUP_CLSFLOWER:
8671                 return i40e_setup_tc_cls_flower(np, type_data);
8672
8673         default:
8674                 return -EOPNOTSUPP;
8675         }
8676 }
8677
8678 static LIST_HEAD(i40e_block_cb_list);
8679
8680 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8681                            void *type_data)
8682 {
8683         struct i40e_netdev_priv *np = netdev_priv(netdev);
8684
8685         switch (type) {
8686         case TC_SETUP_QDISC_MQPRIO:
8687                 return i40e_setup_tc(netdev, type_data);
8688         case TC_SETUP_BLOCK:
8689                 return flow_block_cb_setup_simple(type_data,
8690                                                   &i40e_block_cb_list,
8691                                                   i40e_setup_tc_block_cb,
8692                                                   np, np, true);
8693         default:
8694                 return -EOPNOTSUPP;
8695         }
8696 }
8697
8698 /**
8699  * i40e_open - Called when a network interface is made active
8700  * @netdev: network interface device structure
8701  *
8702  * The open entry point is called when a network interface is made
8703  * active by the system (IFF_UP).  At this point all resources needed
8704  * for transmit and receive operations are allocated, the interrupt
8705  * handler is registered with the OS, the netdev watchdog subtask is
8706  * enabled, and the stack is notified that the interface is ready.
8707  *
8708  * Returns 0 on success, negative value on failure
8709  **/
8710 int i40e_open(struct net_device *netdev)
8711 {
8712         struct i40e_netdev_priv *np = netdev_priv(netdev);
8713         struct i40e_vsi *vsi = np->vsi;
8714         struct i40e_pf *pf = vsi->back;
8715         int err;
8716
8717         /* disallow open during test or if eeprom is broken */
8718         if (test_bit(__I40E_TESTING, pf->state) ||
8719             test_bit(__I40E_BAD_EEPROM, pf->state))
8720                 return -EBUSY;
8721
8722         netif_carrier_off(netdev);
8723
8724         if (i40e_force_link_state(pf, true))
8725                 return -EAGAIN;
8726
8727         err = i40e_vsi_open(vsi);
8728         if (err)
8729                 return err;
8730
8731         /* configure global TSO hardware offload settings */
8732         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8733                                                        TCP_FLAG_FIN) >> 16);
8734         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8735                                                        TCP_FLAG_FIN |
8736                                                        TCP_FLAG_CWR) >> 16);
8737         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8738         udp_tunnel_get_rx_info(netdev);
8739
8740         return 0;
8741 }
8742
8743 /**
8744  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
8745  * @vsi: vsi structure
8746  *
8747  * This updates netdev's number of tx/rx queues
8748  *
8749  * Returns status of setting tx/rx queues
8750  **/
8751 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
8752 {
8753         int ret;
8754
8755         ret = netif_set_real_num_rx_queues(vsi->netdev,
8756                                            vsi->num_queue_pairs);
8757         if (ret)
8758                 return ret;
8759
8760         return netif_set_real_num_tx_queues(vsi->netdev,
8761                                             vsi->num_queue_pairs);
8762 }
8763
8764 /**
8765  * i40e_vsi_open -
8766  * @vsi: the VSI to open
8767  *
8768  * Finish initialization of the VSI.
8769  *
8770  * Returns 0 on success, negative value on failure
8771  *
8772  * Note: expects to be called while under rtnl_lock()
8773  **/
8774 int i40e_vsi_open(struct i40e_vsi *vsi)
8775 {
8776         struct i40e_pf *pf = vsi->back;
8777         char int_name[I40E_INT_NAME_STR_LEN];
8778         int err;
8779
8780         /* allocate descriptors */
8781         err = i40e_vsi_setup_tx_resources(vsi);
8782         if (err)
8783                 goto err_setup_tx;
8784         err = i40e_vsi_setup_rx_resources(vsi);
8785         if (err)
8786                 goto err_setup_rx;
8787
8788         err = i40e_vsi_configure(vsi);
8789         if (err)
8790                 goto err_setup_rx;
8791
8792         if (vsi->netdev) {
8793                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8794                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8795                 err = i40e_vsi_request_irq(vsi, int_name);
8796                 if (err)
8797                         goto err_setup_rx;
8798
8799                 /* Notify the stack of the actual queue counts. */
8800                 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
8801                 if (err)
8802                         goto err_set_queues;
8803
8804         } else if (vsi->type == I40E_VSI_FDIR) {
8805                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8806                          dev_driver_string(&pf->pdev->dev),
8807                          dev_name(&pf->pdev->dev));
8808                 err = i40e_vsi_request_irq(vsi, int_name);
8809                 if (err)
8810                         goto err_setup_rx;
8811
8812         } else {
8813                 err = -EINVAL;
8814                 goto err_setup_rx;
8815         }
8816
8817         err = i40e_up_complete(vsi);
8818         if (err)
8819                 goto err_up_complete;
8820
8821         return 0;
8822
8823 err_up_complete:
8824         i40e_down(vsi);
8825 err_set_queues:
8826         i40e_vsi_free_irq(vsi);
8827 err_setup_rx:
8828         i40e_vsi_free_rx_resources(vsi);
8829 err_setup_tx:
8830         i40e_vsi_free_tx_resources(vsi);
8831         if (vsi == pf->vsi[pf->lan_vsi])
8832                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8833
8834         return err;
8835 }
8836
8837 /**
8838  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
8839  * @pf: Pointer to PF
8840  *
8841  * This function destroys the hlist where all the Flow Director
8842  * filters were saved.
8843  **/
8844 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8845 {
8846         struct i40e_fdir_filter *filter;
8847         struct i40e_flex_pit *pit_entry, *tmp;
8848         struct hlist_node *node2;
8849
8850         hlist_for_each_entry_safe(filter, node2,
8851                                   &pf->fdir_filter_list, fdir_node) {
8852                 hlist_del(&filter->fdir_node);
8853                 kfree(filter);
8854         }
8855
8856         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8857                 list_del(&pit_entry->list);
8858                 kfree(pit_entry);
8859         }
8860         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8861
8862         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8863                 list_del(&pit_entry->list);
8864                 kfree(pit_entry);
8865         }
8866         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8867
8868         pf->fdir_pf_active_filters = 0;
8869         i40e_reset_fdir_filter_cnt(pf);
8870
8871         /* Reprogram the default input set for TCP/IPv4 */
8872         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8873                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8874                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8875
8876         /* Reprogram the default input set for TCP/IPv6 */
8877         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
8878                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8879                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8880
8881         /* Reprogram the default input set for UDP/IPv4 */
8882         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8883                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8884                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8885
8886         /* Reprogram the default input set for UDP/IPv6 */
8887         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
8888                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8889                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8890
8891         /* Reprogram the default input set for SCTP/IPv4 */
8892         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8893                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8894                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8895
8896         /* Reprogram the default input set for SCTP/IPv6 */
8897         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
8898                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8899                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8900
8901         /* Reprogram the default input set for Other/IPv4 */
8902         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8903                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8904
8905         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8906                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8907
8908         /* Reprogram the default input set for Other/IPv6 */
8909         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
8910                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8911
8912         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
8913                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8914 }
8915
8916 /**
8917  * i40e_cloud_filter_exit - Cleans up the cloud filters
8918  * @pf: Pointer to PF
8919  *
8920  * This function destroys the hlist where all the cloud filters
8921  * were saved.
8922  **/
8923 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8924 {
8925         struct i40e_cloud_filter *cfilter;
8926         struct hlist_node *node;
8927
8928         hlist_for_each_entry_safe(cfilter, node,
8929                                   &pf->cloud_filter_list, cloud_node) {
8930                 hlist_del(&cfilter->cloud_node);
8931                 kfree(cfilter);
8932         }
8933         pf->num_cloud_filters = 0;
8934
8935         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8936             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8937                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8938                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8939                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8940         }
8941 }
8942
8943 /**
8944  * i40e_close - Disables a network interface
8945  * @netdev: network interface device structure
8946  *
8947  * The close entry point is called when an interface is de-activated
8948  * by the OS.  The hardware is still under the driver's control, but
8949  * this netdev interface is disabled.
8950  *
8951  * Returns 0, this is not allowed to fail
8952  **/
8953 int i40e_close(struct net_device *netdev)
8954 {
8955         struct i40e_netdev_priv *np = netdev_priv(netdev);
8956         struct i40e_vsi *vsi = np->vsi;
8957
8958         i40e_vsi_close(vsi);
8959
8960         return 0;
8961 }
8962
8963 /**
8964  * i40e_do_reset - Start a PF or Core Reset sequence
8965  * @pf: board private structure
8966  * @reset_flags: which reset is requested
8967  * @lock_acquired: indicates whether or not the lock has been acquired
8968  * before this function was called.
8969  *
8970  * The essential difference in resets is that the PF Reset
8971  * doesn't clear the packet buffers, doesn't reset the PE
8972  * firmware, and doesn't bother the other PFs on the chip.
8973  **/
8974 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8975 {
8976         u32 val;
8977
8978         /* do the biggest reset indicated */
8979         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8980
8981                 /* Request a Global Reset
8982                  *
8983                  * This will start the chip's countdown to the actual full
8984                  * chip reset event, and a warning interrupt to be sent
8985                  * to all PFs, including the requestor.  Our handler
8986                  * for the warning interrupt will deal with the shutdown
8987                  * and recovery of the switch setup.
8988                  */
8989                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8990                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8991                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8992                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8993
8994         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8995
8996                 /* Request a Core Reset
8997                  *
8998                  * Same as Global Reset, except does *not* include the MAC/PHY
8999                  */
9000                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9001                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9002                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
9003                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9004                 i40e_flush(&pf->hw);
9005
9006         } else if (reset_flags & I40E_PF_RESET_FLAG) {
9007
9008                 /* Request a PF Reset
9009                  *
9010                  * Resets only the PF-specific registers
9011                  *
9012                  * This goes directly to the tear-down and rebuild of
9013                  * the switch, since we need to do all the recovery as
9014                  * for the Core Reset.
9015                  */
9016                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
9017                 i40e_handle_reset_warning(pf, lock_acquired);
9018
9019         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9020                 /* Request a PF Reset
9021                  *
9022                  * Resets PF and reinitializes PFs VSI.
9023                  */
9024                 i40e_prep_for_reset(pf);
9025                 i40e_reset_and_rebuild(pf, true, lock_acquired);
9026                 dev_info(&pf->pdev->dev,
9027                          pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
9028                          "FW LLDP is disabled\n" :
9029                          "FW LLDP is enabled\n");
9030
9031         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9032                 int v;
9033
9034                 /* Find the VSI(s) that requested a re-init */
9035                 dev_info(&pf->pdev->dev,
9036                          "VSI reinit requested\n");
9037                 for (v = 0; v < pf->num_alloc_vsi; v++) {
9038                         struct i40e_vsi *vsi = pf->vsi[v];
9039
9040                         if (vsi != NULL &&
9041                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9042                                                vsi->state))
9043                                 i40e_vsi_reinit_locked(pf->vsi[v]);
9044                 }
9045         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9046                 int v;
9047
9048                 /* Find the VSI(s) that needs to be brought down */
9049                 dev_info(&pf->pdev->dev, "VSI down requested\n");
9050                 for (v = 0; v < pf->num_alloc_vsi; v++) {
9051                         struct i40e_vsi *vsi = pf->vsi[v];
9052
9053                         if (vsi != NULL &&
9054                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9055                                                vsi->state)) {
9056                                 set_bit(__I40E_VSI_DOWN, vsi->state);
9057                                 i40e_down(vsi);
9058                         }
9059                 }
9060         } else {
9061                 dev_info(&pf->pdev->dev,
9062                          "bad reset request 0x%08x\n", reset_flags);
9063         }
9064 }
9065
9066 #ifdef CONFIG_I40E_DCB
9067 /**
9068  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9069  * @pf: board private structure
9070  * @old_cfg: current DCB config
9071  * @new_cfg: new DCB config
9072  **/
9073 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9074                             struct i40e_dcbx_config *old_cfg,
9075                             struct i40e_dcbx_config *new_cfg)
9076 {
9077         bool need_reconfig = false;
9078
9079         /* Check if ETS configuration has changed */
9080         if (memcmp(&new_cfg->etscfg,
9081                    &old_cfg->etscfg,
9082                    sizeof(new_cfg->etscfg))) {
9083                 /* If Priority Table has changed reconfig is needed */
9084                 if (memcmp(&new_cfg->etscfg.prioritytable,
9085                            &old_cfg->etscfg.prioritytable,
9086                            sizeof(new_cfg->etscfg.prioritytable))) {
9087                         need_reconfig = true;
9088                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9089                 }
9090
9091                 if (memcmp(&new_cfg->etscfg.tcbwtable,
9092                            &old_cfg->etscfg.tcbwtable,
9093                            sizeof(new_cfg->etscfg.tcbwtable)))
9094                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9095
9096                 if (memcmp(&new_cfg->etscfg.tsatable,
9097                            &old_cfg->etscfg.tsatable,
9098                            sizeof(new_cfg->etscfg.tsatable)))
9099                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9100         }
9101
9102         /* Check if PFC configuration has changed */
9103         if (memcmp(&new_cfg->pfc,
9104                    &old_cfg->pfc,
9105                    sizeof(new_cfg->pfc))) {
9106                 need_reconfig = true;
9107                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9108         }
9109
9110         /* Check if APP Table has changed */
9111         if (memcmp(&new_cfg->app,
9112                    &old_cfg->app,
9113                    sizeof(new_cfg->app))) {
9114                 need_reconfig = true;
9115                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9116         }
9117
9118         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9119         return need_reconfig;
9120 }
9121
9122 /**
9123  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9124  * @pf: board private structure
9125  * @e: event info posted on ARQ
9126  **/
9127 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9128                                   struct i40e_arq_event_info *e)
9129 {
9130         struct i40e_aqc_lldp_get_mib *mib =
9131                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9132         struct i40e_hw *hw = &pf->hw;
9133         struct i40e_dcbx_config tmp_dcbx_cfg;
9134         bool need_reconfig = false;
9135         int ret = 0;
9136         u8 type;
9137
9138         /* X710-T*L 2.5G and 5G speeds don't support DCB */
9139         if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9140             (hw->phy.link_info.link_speed &
9141              ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9142              !(pf->flags & I40E_FLAG_DCB_CAPABLE))
9143                 /* let firmware decide if the DCB should be disabled */
9144                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
9145
9146         /* Not DCB capable or capability disabled */
9147         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
9148                 return ret;
9149
9150         /* Ignore if event is not for Nearest Bridge */
9151         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9152                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9153         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9154         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9155                 return ret;
9156
9157         /* Check MIB Type and return if event for Remote MIB update */
9158         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9159         dev_dbg(&pf->pdev->dev,
9160                 "LLDP event mib type %s\n", type ? "remote" : "local");
9161         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9162                 /* Update the remote cached instance and return */
9163                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9164                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9165                                 &hw->remote_dcbx_config);
9166                 goto exit;
9167         }
9168
9169         /* Store the old configuration */
9170         tmp_dcbx_cfg = hw->local_dcbx_config;
9171
9172         /* Reset the old DCBx configuration data */
9173         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9174         /* Get updated DCBX data from firmware */
9175         ret = i40e_get_dcb_config(&pf->hw);
9176         if (ret) {
9177                 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9178                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9179                     (hw->phy.link_info.link_speed &
9180                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9181                         dev_warn(&pf->pdev->dev,
9182                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9183                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9184                 } else {
9185                         dev_info(&pf->pdev->dev,
9186                                  "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
9187                                  i40e_stat_str(&pf->hw, ret),
9188                                  i40e_aq_str(&pf->hw,
9189                                              pf->hw.aq.asq_last_status));
9190                 }
9191                 goto exit;
9192         }
9193
9194         /* No change detected in DCBX configs */
9195         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9196                     sizeof(tmp_dcbx_cfg))) {
9197                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9198                 goto exit;
9199         }
9200
9201         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9202                                                &hw->local_dcbx_config);
9203
9204         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9205
9206         if (!need_reconfig)
9207                 goto exit;
9208
9209         /* Enable DCB tagging only when more than one TC */
9210         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9211                 pf->flags |= I40E_FLAG_DCB_ENABLED;
9212         else
9213                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9214
9215         set_bit(__I40E_PORT_SUSPENDED, pf->state);
9216         /* Reconfiguration needed quiesce all VSIs */
9217         i40e_pf_quiesce_all_vsi(pf);
9218
9219         /* Changes in configuration update VEB/VSI */
9220         i40e_dcb_reconfigure(pf);
9221
9222         ret = i40e_resume_port_tx(pf);
9223
9224         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9225         /* In case of error no point in resuming VSIs */
9226         if (ret)
9227                 goto exit;
9228
9229         /* Wait for the PF's queues to be disabled */
9230         ret = i40e_pf_wait_queues_disabled(pf);
9231         if (ret) {
9232                 /* Schedule PF reset to recover */
9233                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9234                 i40e_service_event_schedule(pf);
9235         } else {
9236                 i40e_pf_unquiesce_all_vsi(pf);
9237                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9238                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9239         }
9240
9241 exit:
9242         return ret;
9243 }
9244 #endif /* CONFIG_I40E_DCB */
9245
9246 /**
9247  * i40e_do_reset_safe - Protected reset path for userland calls.
9248  * @pf: board private structure
9249  * @reset_flags: which reset is requested
9250  *
9251  **/
9252 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9253 {
9254         rtnl_lock();
9255         i40e_do_reset(pf, reset_flags, true);
9256         rtnl_unlock();
9257 }
9258
9259 /**
9260  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9261  * @pf: board private structure
9262  * @e: event info posted on ARQ
9263  *
9264  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9265  * and VF queues
9266  **/
9267 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9268                                            struct i40e_arq_event_info *e)
9269 {
9270         struct i40e_aqc_lan_overflow *data =
9271                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9272         u32 queue = le32_to_cpu(data->prtdcb_rupto);
9273         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9274         struct i40e_hw *hw = &pf->hw;
9275         struct i40e_vf *vf;
9276         u16 vf_id;
9277
9278         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9279                 queue, qtx_ctl);
9280
9281         /* Queue belongs to VF, find the VF and issue VF reset */
9282         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9283             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9284                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9285                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9286                 vf_id -= hw->func_caps.vf_base_id;
9287                 vf = &pf->vf[vf_id];
9288                 i40e_vc_notify_vf_reset(vf);
9289                 /* Allow VF to process pending reset notification */
9290                 msleep(20);
9291                 i40e_reset_vf(vf, false);
9292         }
9293 }
9294
9295 /**
9296  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9297  * @pf: board private structure
9298  **/
9299 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9300 {
9301         u32 val, fcnt_prog;
9302
9303         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9304         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9305         return fcnt_prog;
9306 }
9307
9308 /**
9309  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9310  * @pf: board private structure
9311  **/
9312 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9313 {
9314         u32 val, fcnt_prog;
9315
9316         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9317         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9318                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9319                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9320         return fcnt_prog;
9321 }
9322
9323 /**
9324  * i40e_get_global_fd_count - Get total FD filters programmed on device
9325  * @pf: board private structure
9326  **/
9327 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9328 {
9329         u32 val, fcnt_prog;
9330
9331         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9332         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9333                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9334                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9335         return fcnt_prog;
9336 }
9337
9338 /**
9339  * i40e_reenable_fdir_sb - Restore FDir SB capability
9340  * @pf: board private structure
9341  **/
9342 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9343 {
9344         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9345                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9346                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9347                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9348 }
9349
9350 /**
9351  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9352  * @pf: board private structure
9353  **/
9354 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9355 {
9356         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9357                 /* ATR uses the same filtering logic as SB rules. It only
9358                  * functions properly if the input set mask is at the default
9359                  * settings. It is safe to restore the default input set
9360                  * because there are no active TCPv4 filter rules.
9361                  */
9362                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9363                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9364                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9365
9366                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9367                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9368                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9369         }
9370 }
9371
9372 /**
9373  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9374  * @pf: board private structure
9375  * @filter: FDir filter to remove
9376  */
9377 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9378                                        struct i40e_fdir_filter *filter)
9379 {
9380         /* Update counters */
9381         pf->fdir_pf_active_filters--;
9382         pf->fd_inv = 0;
9383
9384         switch (filter->flow_type) {
9385         case TCP_V4_FLOW:
9386                 pf->fd_tcp4_filter_cnt--;
9387                 break;
9388         case UDP_V4_FLOW:
9389                 pf->fd_udp4_filter_cnt--;
9390                 break;
9391         case SCTP_V4_FLOW:
9392                 pf->fd_sctp4_filter_cnt--;
9393                 break;
9394         case TCP_V6_FLOW:
9395                 pf->fd_tcp6_filter_cnt--;
9396                 break;
9397         case UDP_V6_FLOW:
9398                 pf->fd_udp6_filter_cnt--;
9399                 break;
9400         case SCTP_V6_FLOW:
9401                 pf->fd_udp6_filter_cnt--;
9402                 break;
9403         case IP_USER_FLOW:
9404                 switch (filter->ipl4_proto) {
9405                 case IPPROTO_TCP:
9406                         pf->fd_tcp4_filter_cnt--;
9407                         break;
9408                 case IPPROTO_UDP:
9409                         pf->fd_udp4_filter_cnt--;
9410                         break;
9411                 case IPPROTO_SCTP:
9412                         pf->fd_sctp4_filter_cnt--;
9413                         break;
9414                 case IPPROTO_IP:
9415                         pf->fd_ip4_filter_cnt--;
9416                         break;
9417                 }
9418                 break;
9419         case IPV6_USER_FLOW:
9420                 switch (filter->ipl4_proto) {
9421                 case IPPROTO_TCP:
9422                         pf->fd_tcp6_filter_cnt--;
9423                         break;
9424                 case IPPROTO_UDP:
9425                         pf->fd_udp6_filter_cnt--;
9426                         break;
9427                 case IPPROTO_SCTP:
9428                         pf->fd_sctp6_filter_cnt--;
9429                         break;
9430                 case IPPROTO_IP:
9431                         pf->fd_ip6_filter_cnt--;
9432                         break;
9433                 }
9434                 break;
9435         }
9436
9437         /* Remove the filter from the list and free memory */
9438         hlist_del(&filter->fdir_node);
9439         kfree(filter);
9440 }
9441
9442 /**
9443  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9444  * @pf: board private structure
9445  **/
9446 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9447 {
9448         struct i40e_fdir_filter *filter;
9449         u32 fcnt_prog, fcnt_avail;
9450         struct hlist_node *node;
9451
9452         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9453                 return;
9454
9455         /* Check if we have enough room to re-enable FDir SB capability. */
9456         fcnt_prog = i40e_get_global_fd_count(pf);
9457         fcnt_avail = pf->fdir_pf_filter_count;
9458         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9459             (pf->fd_add_err == 0) ||
9460             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9461                 i40e_reenable_fdir_sb(pf);
9462
9463         /* We should wait for even more space before re-enabling ATR.
9464          * Additionally, we cannot enable ATR as long as we still have TCP SB
9465          * rules active.
9466          */
9467         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9468             pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9469                 i40e_reenable_fdir_atr(pf);
9470
9471         /* if hw had a problem adding a filter, delete it */
9472         if (pf->fd_inv > 0) {
9473                 hlist_for_each_entry_safe(filter, node,
9474                                           &pf->fdir_filter_list, fdir_node)
9475                         if (filter->fd_id == pf->fd_inv)
9476                                 i40e_delete_invalid_filter(pf, filter);
9477         }
9478 }
9479
9480 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9481 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9482 /**
9483  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9484  * @pf: board private structure
9485  **/
9486 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9487 {
9488         unsigned long min_flush_time;
9489         int flush_wait_retry = 50;
9490         bool disable_atr = false;
9491         int fd_room;
9492         int reg;
9493
9494         if (!time_after(jiffies, pf->fd_flush_timestamp +
9495                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9496                 return;
9497
9498         /* If the flush is happening too quick and we have mostly SB rules we
9499          * should not re-enable ATR for some time.
9500          */
9501         min_flush_time = pf->fd_flush_timestamp +
9502                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9503         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9504
9505         if (!(time_after(jiffies, min_flush_time)) &&
9506             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9507                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9508                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9509                 disable_atr = true;
9510         }
9511
9512         pf->fd_flush_timestamp = jiffies;
9513         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9514         /* flush all filters */
9515         wr32(&pf->hw, I40E_PFQF_CTL_1,
9516              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9517         i40e_flush(&pf->hw);
9518         pf->fd_flush_cnt++;
9519         pf->fd_add_err = 0;
9520         do {
9521                 /* Check FD flush status every 5-6msec */
9522                 usleep_range(5000, 6000);
9523                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9524                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9525                         break;
9526         } while (flush_wait_retry--);
9527         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9528                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9529         } else {
9530                 /* replay sideband filters */
9531                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9532                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9533                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9534                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9535                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9536                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9537         }
9538 }
9539
9540 /**
9541  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9542  * @pf: board private structure
9543  **/
9544 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9545 {
9546         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9547 }
9548
9549 /**
9550  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9551  * @pf: board private structure
9552  **/
9553 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9554 {
9555
9556         /* if interface is down do nothing */
9557         if (test_bit(__I40E_DOWN, pf->state))
9558                 return;
9559
9560         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9561                 i40e_fdir_flush_and_replay(pf);
9562
9563         i40e_fdir_check_and_reenable(pf);
9564
9565 }
9566
9567 /**
9568  * i40e_vsi_link_event - notify VSI of a link event
9569  * @vsi: vsi to be notified
9570  * @link_up: link up or down
9571  **/
9572 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9573 {
9574         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9575                 return;
9576
9577         switch (vsi->type) {
9578         case I40E_VSI_MAIN:
9579                 if (!vsi->netdev || !vsi->netdev_registered)
9580                         break;
9581
9582                 if (link_up) {
9583                         netif_carrier_on(vsi->netdev);
9584                         netif_tx_wake_all_queues(vsi->netdev);
9585                 } else {
9586                         netif_carrier_off(vsi->netdev);
9587                         netif_tx_stop_all_queues(vsi->netdev);
9588                 }
9589                 break;
9590
9591         case I40E_VSI_SRIOV:
9592         case I40E_VSI_VMDQ2:
9593         case I40E_VSI_CTRL:
9594         case I40E_VSI_IWARP:
9595         case I40E_VSI_MIRROR:
9596         default:
9597                 /* there is no notification for other VSIs */
9598                 break;
9599         }
9600 }
9601
9602 /**
9603  * i40e_veb_link_event - notify elements on the veb of a link event
9604  * @veb: veb to be notified
9605  * @link_up: link up or down
9606  **/
9607 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9608 {
9609         struct i40e_pf *pf;
9610         int i;
9611
9612         if (!veb || !veb->pf)
9613                 return;
9614         pf = veb->pf;
9615
9616         /* depth first... */
9617         for (i = 0; i < I40E_MAX_VEB; i++)
9618                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9619                         i40e_veb_link_event(pf->veb[i], link_up);
9620
9621         /* ... now the local VSIs */
9622         for (i = 0; i < pf->num_alloc_vsi; i++)
9623                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9624                         i40e_vsi_link_event(pf->vsi[i], link_up);
9625 }
9626
9627 /**
9628  * i40e_link_event - Update netif_carrier status
9629  * @pf: board private structure
9630  **/
9631 static void i40e_link_event(struct i40e_pf *pf)
9632 {
9633         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9634         u8 new_link_speed, old_link_speed;
9635         i40e_status status;
9636         bool new_link, old_link;
9637 #ifdef CONFIG_I40E_DCB
9638         int err;
9639 #endif /* CONFIG_I40E_DCB */
9640
9641         /* set this to force the get_link_status call to refresh state */
9642         pf->hw.phy.get_link_info = true;
9643         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9644         status = i40e_get_link_status(&pf->hw, &new_link);
9645
9646         /* On success, disable temp link polling */
9647         if (status == I40E_SUCCESS) {
9648                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9649         } else {
9650                 /* Enable link polling temporarily until i40e_get_link_status
9651                  * returns I40E_SUCCESS
9652                  */
9653                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9654                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9655                         status);
9656                 return;
9657         }
9658
9659         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9660         new_link_speed = pf->hw.phy.link_info.link_speed;
9661
9662         if (new_link == old_link &&
9663             new_link_speed == old_link_speed &&
9664             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9665              new_link == netif_carrier_ok(vsi->netdev)))
9666                 return;
9667
9668         i40e_print_link_message(vsi, new_link);
9669
9670         /* Notify the base of the switch tree connected to
9671          * the link.  Floating VEBs are not notified.
9672          */
9673         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9674                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9675         else
9676                 i40e_vsi_link_event(vsi, new_link);
9677
9678         if (pf->vf)
9679                 i40e_vc_notify_link_state(pf);
9680
9681         if (pf->flags & I40E_FLAG_PTP)
9682                 i40e_ptp_set_increment(pf);
9683 #ifdef CONFIG_I40E_DCB
9684         if (new_link == old_link)
9685                 return;
9686         /* Not SW DCB so firmware will take care of default settings */
9687         if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9688                 return;
9689
9690         /* We cover here only link down, as after link up in case of SW DCB
9691          * SW LLDP agent will take care of setting it up
9692          */
9693         if (!new_link) {
9694                 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9695                 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9696                 err = i40e_dcb_sw_default_config(pf);
9697                 if (err) {
9698                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
9699                                        I40E_FLAG_DCB_ENABLED);
9700                 } else {
9701                         pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9702                                        DCB_CAP_DCBX_VER_IEEE;
9703                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
9704                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9705                 }
9706         }
9707 #endif /* CONFIG_I40E_DCB */
9708 }
9709
9710 /**
9711  * i40e_watchdog_subtask - periodic checks not using event driven response
9712  * @pf: board private structure
9713  **/
9714 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9715 {
9716         int i;
9717
9718         /* if interface is down do nothing */
9719         if (test_bit(__I40E_DOWN, pf->state) ||
9720             test_bit(__I40E_CONFIG_BUSY, pf->state))
9721                 return;
9722
9723         /* make sure we don't do these things too often */
9724         if (time_before(jiffies, (pf->service_timer_previous +
9725                                   pf->service_timer_period)))
9726                 return;
9727         pf->service_timer_previous = jiffies;
9728
9729         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9730             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9731                 i40e_link_event(pf);
9732
9733         /* Update the stats for active netdevs so the network stack
9734          * can look at updated numbers whenever it cares to
9735          */
9736         for (i = 0; i < pf->num_alloc_vsi; i++)
9737                 if (pf->vsi[i] && pf->vsi[i]->netdev)
9738                         i40e_update_stats(pf->vsi[i]);
9739
9740         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9741                 /* Update the stats for the active switching components */
9742                 for (i = 0; i < I40E_MAX_VEB; i++)
9743                         if (pf->veb[i])
9744                                 i40e_update_veb_stats(pf->veb[i]);
9745         }
9746
9747         i40e_ptp_rx_hang(pf);
9748         i40e_ptp_tx_hang(pf);
9749 }
9750
9751 /**
9752  * i40e_reset_subtask - Set up for resetting the device and driver
9753  * @pf: board private structure
9754  **/
9755 static void i40e_reset_subtask(struct i40e_pf *pf)
9756 {
9757         u32 reset_flags = 0;
9758
9759         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9760                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9761                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9762         }
9763         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9764                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9765                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9766         }
9767         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9768                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9769                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9770         }
9771         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9772                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9773                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9774         }
9775         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9776                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9777                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9778         }
9779
9780         /* If there's a recovery already waiting, it takes
9781          * precedence before starting a new reset sequence.
9782          */
9783         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9784                 i40e_prep_for_reset(pf);
9785                 i40e_reset(pf);
9786                 i40e_rebuild(pf, false, false);
9787         }
9788
9789         /* If we're already down or resetting, just bail */
9790         if (reset_flags &&
9791             !test_bit(__I40E_DOWN, pf->state) &&
9792             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9793                 i40e_do_reset(pf, reset_flags, false);
9794         }
9795 }
9796
9797 /**
9798  * i40e_handle_link_event - Handle link event
9799  * @pf: board private structure
9800  * @e: event info posted on ARQ
9801  **/
9802 static void i40e_handle_link_event(struct i40e_pf *pf,
9803                                    struct i40e_arq_event_info *e)
9804 {
9805         struct i40e_aqc_get_link_status *status =
9806                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9807
9808         /* Do a new status request to re-enable LSE reporting
9809          * and load new status information into the hw struct
9810          * This completely ignores any state information
9811          * in the ARQ event info, instead choosing to always
9812          * issue the AQ update link status command.
9813          */
9814         i40e_link_event(pf);
9815
9816         /* Check if module meets thermal requirements */
9817         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9818                 dev_err(&pf->pdev->dev,
9819                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9820                 dev_err(&pf->pdev->dev,
9821                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9822         } else {
9823                 /* check for unqualified module, if link is down, suppress
9824                  * the message if link was forced to be down.
9825                  */
9826                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9827                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9828                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
9829                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9830                         dev_err(&pf->pdev->dev,
9831                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9832                         dev_err(&pf->pdev->dev,
9833                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9834                 }
9835         }
9836 }
9837
9838 /**
9839  * i40e_clean_adminq_subtask - Clean the AdminQ rings
9840  * @pf: board private structure
9841  **/
9842 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9843 {
9844         struct i40e_arq_event_info event;
9845         struct i40e_hw *hw = &pf->hw;
9846         u16 pending, i = 0;
9847         i40e_status ret;
9848         u16 opcode;
9849         u32 oldval;
9850         u32 val;
9851
9852         /* Do not run clean AQ when PF reset fails */
9853         if (test_bit(__I40E_RESET_FAILED, pf->state))
9854                 return;
9855
9856         /* check for error indications */
9857         val = rd32(&pf->hw, pf->hw.aq.arq.len);
9858         oldval = val;
9859         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9860                 if (hw->debug_mask & I40E_DEBUG_AQ)
9861                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9862                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9863         }
9864         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9865                 if (hw->debug_mask & I40E_DEBUG_AQ)
9866                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9867                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9868                 pf->arq_overflows++;
9869         }
9870         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9871                 if (hw->debug_mask & I40E_DEBUG_AQ)
9872                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9873                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9874         }
9875         if (oldval != val)
9876                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9877
9878         val = rd32(&pf->hw, pf->hw.aq.asq.len);
9879         oldval = val;
9880         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9881                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9882                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9883                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9884         }
9885         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9886                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9887                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9888                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9889         }
9890         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9891                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9892                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9893                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9894         }
9895         if (oldval != val)
9896                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9897
9898         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9899         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9900         if (!event.msg_buf)
9901                 return;
9902
9903         do {
9904                 ret = i40e_clean_arq_element(hw, &event, &pending);
9905                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9906                         break;
9907                 else if (ret) {
9908                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9909                         break;
9910                 }
9911
9912                 opcode = le16_to_cpu(event.desc.opcode);
9913                 switch (opcode) {
9914
9915                 case i40e_aqc_opc_get_link_status:
9916                         rtnl_lock();
9917                         i40e_handle_link_event(pf, &event);
9918                         rtnl_unlock();
9919                         break;
9920                 case i40e_aqc_opc_send_msg_to_pf:
9921                         ret = i40e_vc_process_vf_msg(pf,
9922                                         le16_to_cpu(event.desc.retval),
9923                                         le32_to_cpu(event.desc.cookie_high),
9924                                         le32_to_cpu(event.desc.cookie_low),
9925                                         event.msg_buf,
9926                                         event.msg_len);
9927                         break;
9928                 case i40e_aqc_opc_lldp_update_mib:
9929                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9930 #ifdef CONFIG_I40E_DCB
9931                         rtnl_lock();
9932                         i40e_handle_lldp_event(pf, &event);
9933                         rtnl_unlock();
9934 #endif /* CONFIG_I40E_DCB */
9935                         break;
9936                 case i40e_aqc_opc_event_lan_overflow:
9937                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9938                         i40e_handle_lan_overflow_event(pf, &event);
9939                         break;
9940                 case i40e_aqc_opc_send_msg_to_peer:
9941                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9942                         break;
9943                 case i40e_aqc_opc_nvm_erase:
9944                 case i40e_aqc_opc_nvm_update:
9945                 case i40e_aqc_opc_oem_post_update:
9946                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9947                                    "ARQ NVM operation 0x%04x completed\n",
9948                                    opcode);
9949                         break;
9950                 default:
9951                         dev_info(&pf->pdev->dev,
9952                                  "ARQ: Unknown event 0x%04x ignored\n",
9953                                  opcode);
9954                         break;
9955                 }
9956         } while (i++ < pf->adminq_work_limit);
9957
9958         if (i < pf->adminq_work_limit)
9959                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9960
9961         /* re-enable Admin queue interrupt cause */
9962         val = rd32(hw, I40E_PFINT_ICR0_ENA);
9963         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9964         wr32(hw, I40E_PFINT_ICR0_ENA, val);
9965         i40e_flush(hw);
9966
9967         kfree(event.msg_buf);
9968 }
9969
9970 /**
9971  * i40e_verify_eeprom - make sure eeprom is good to use
9972  * @pf: board private structure
9973  **/
9974 static void i40e_verify_eeprom(struct i40e_pf *pf)
9975 {
9976         int err;
9977
9978         err = i40e_diag_eeprom_test(&pf->hw);
9979         if (err) {
9980                 /* retry in case of garbage read */
9981                 err = i40e_diag_eeprom_test(&pf->hw);
9982                 if (err) {
9983                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9984                                  err);
9985                         set_bit(__I40E_BAD_EEPROM, pf->state);
9986                 }
9987         }
9988
9989         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9990                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9991                 clear_bit(__I40E_BAD_EEPROM, pf->state);
9992         }
9993 }
9994
9995 /**
9996  * i40e_enable_pf_switch_lb
9997  * @pf: pointer to the PF structure
9998  *
9999  * enable switch loop back or die - no point in a return value
10000  **/
10001 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10002 {
10003         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10004         struct i40e_vsi_context ctxt;
10005         int ret;
10006
10007         ctxt.seid = pf->main_vsi_seid;
10008         ctxt.pf_num = pf->hw.pf_id;
10009         ctxt.vf_num = 0;
10010         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10011         if (ret) {
10012                 dev_info(&pf->pdev->dev,
10013                          "couldn't get PF vsi config, err %s aq_err %s\n",
10014                          i40e_stat_str(&pf->hw, ret),
10015                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10016                 return;
10017         }
10018         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10019         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10020         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10021
10022         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10023         if (ret) {
10024                 dev_info(&pf->pdev->dev,
10025                          "update vsi switch failed, err %s aq_err %s\n",
10026                          i40e_stat_str(&pf->hw, ret),
10027                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10028         }
10029 }
10030
10031 /**
10032  * i40e_disable_pf_switch_lb
10033  * @pf: pointer to the PF structure
10034  *
10035  * disable switch loop back or die - no point in a return value
10036  **/
10037 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10038 {
10039         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10040         struct i40e_vsi_context ctxt;
10041         int ret;
10042
10043         ctxt.seid = pf->main_vsi_seid;
10044         ctxt.pf_num = pf->hw.pf_id;
10045         ctxt.vf_num = 0;
10046         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10047         if (ret) {
10048                 dev_info(&pf->pdev->dev,
10049                          "couldn't get PF vsi config, err %s aq_err %s\n",
10050                          i40e_stat_str(&pf->hw, ret),
10051                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10052                 return;
10053         }
10054         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10055         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10056         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10057
10058         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10059         if (ret) {
10060                 dev_info(&pf->pdev->dev,
10061                          "update vsi switch failed, err %s aq_err %s\n",
10062                          i40e_stat_str(&pf->hw, ret),
10063                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10064         }
10065 }
10066
10067 /**
10068  * i40e_config_bridge_mode - Configure the HW bridge mode
10069  * @veb: pointer to the bridge instance
10070  *
10071  * Configure the loop back mode for the LAN VSI that is downlink to the
10072  * specified HW bridge instance. It is expected this function is called
10073  * when a new HW bridge is instantiated.
10074  **/
10075 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10076 {
10077         struct i40e_pf *pf = veb->pf;
10078
10079         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10080                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10081                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10082         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10083                 i40e_disable_pf_switch_lb(pf);
10084         else
10085                 i40e_enable_pf_switch_lb(pf);
10086 }
10087
10088 /**
10089  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
10090  * @veb: pointer to the VEB instance
10091  *
10092  * This is a recursive function that first builds the attached VSIs then
10093  * recurses in to build the next layer of VEB.  We track the connections
10094  * through our own index numbers because the seid's from the HW could
10095  * change across the reset.
10096  **/
10097 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10098 {
10099         struct i40e_vsi *ctl_vsi = NULL;
10100         struct i40e_pf *pf = veb->pf;
10101         int v, veb_idx;
10102         int ret;
10103
10104         /* build VSI that owns this VEB, temporarily attached to base VEB */
10105         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
10106                 if (pf->vsi[v] &&
10107                     pf->vsi[v]->veb_idx == veb->idx &&
10108                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
10109                         ctl_vsi = pf->vsi[v];
10110                         break;
10111                 }
10112         }
10113         if (!ctl_vsi) {
10114                 dev_info(&pf->pdev->dev,
10115                          "missing owner VSI for veb_idx %d\n", veb->idx);
10116                 ret = -ENOENT;
10117                 goto end_reconstitute;
10118         }
10119         if (ctl_vsi != pf->vsi[pf->lan_vsi])
10120                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10121         ret = i40e_add_vsi(ctl_vsi);
10122         if (ret) {
10123                 dev_info(&pf->pdev->dev,
10124                          "rebuild of veb_idx %d owner VSI failed: %d\n",
10125                          veb->idx, ret);
10126                 goto end_reconstitute;
10127         }
10128         i40e_vsi_reset_stats(ctl_vsi);
10129
10130         /* create the VEB in the switch and move the VSI onto the VEB */
10131         ret = i40e_add_veb(veb, ctl_vsi);
10132         if (ret)
10133                 goto end_reconstitute;
10134
10135         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10136                 veb->bridge_mode = BRIDGE_MODE_VEB;
10137         else
10138                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10139         i40e_config_bridge_mode(veb);
10140
10141         /* create the remaining VSIs attached to this VEB */
10142         for (v = 0; v < pf->num_alloc_vsi; v++) {
10143                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
10144                         continue;
10145
10146                 if (pf->vsi[v]->veb_idx == veb->idx) {
10147                         struct i40e_vsi *vsi = pf->vsi[v];
10148
10149                         vsi->uplink_seid = veb->seid;
10150                         ret = i40e_add_vsi(vsi);
10151                         if (ret) {
10152                                 dev_info(&pf->pdev->dev,
10153                                          "rebuild of vsi_idx %d failed: %d\n",
10154                                          v, ret);
10155                                 goto end_reconstitute;
10156                         }
10157                         i40e_vsi_reset_stats(vsi);
10158                 }
10159         }
10160
10161         /* create any VEBs attached to this VEB - RECURSION */
10162         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10163                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
10164                         pf->veb[veb_idx]->uplink_seid = veb->seid;
10165                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
10166                         if (ret)
10167                                 break;
10168                 }
10169         }
10170
10171 end_reconstitute:
10172         return ret;
10173 }
10174
10175 /**
10176  * i40e_get_capabilities - get info about the HW
10177  * @pf: the PF struct
10178  * @list_type: AQ capability to be queried
10179  **/
10180 static int i40e_get_capabilities(struct i40e_pf *pf,
10181                                  enum i40e_admin_queue_opc list_type)
10182 {
10183         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10184         u16 data_size;
10185         int buf_len;
10186         int err;
10187
10188         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10189         do {
10190                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10191                 if (!cap_buf)
10192                         return -ENOMEM;
10193
10194                 /* this loads the data into the hw struct for us */
10195                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10196                                                     &data_size, list_type,
10197                                                     NULL);
10198                 /* data loaded, buffer no longer needed */
10199                 kfree(cap_buf);
10200
10201                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10202                         /* retry with a larger buffer */
10203                         buf_len = data_size;
10204                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10205                         dev_info(&pf->pdev->dev,
10206                                  "capability discovery failed, err %s aq_err %s\n",
10207                                  i40e_stat_str(&pf->hw, err),
10208                                  i40e_aq_str(&pf->hw,
10209                                              pf->hw.aq.asq_last_status));
10210                         return -ENODEV;
10211                 }
10212         } while (err);
10213
10214         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10215                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10216                         dev_info(&pf->pdev->dev,
10217                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10218                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10219                                  pf->hw.func_caps.num_msix_vectors,
10220                                  pf->hw.func_caps.num_msix_vectors_vf,
10221                                  pf->hw.func_caps.fd_filters_guaranteed,
10222                                  pf->hw.func_caps.fd_filters_best_effort,
10223                                  pf->hw.func_caps.num_tx_qp,
10224                                  pf->hw.func_caps.num_vsis);
10225                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10226                         dev_info(&pf->pdev->dev,
10227                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
10228                                  pf->hw.dev_caps.switch_mode,
10229                                  pf->hw.dev_caps.valid_functions);
10230                         dev_info(&pf->pdev->dev,
10231                                  "SR-IOV=%d, num_vfs for all function=%u\n",
10232                                  pf->hw.dev_caps.sr_iov_1_1,
10233                                  pf->hw.dev_caps.num_vfs);
10234                         dev_info(&pf->pdev->dev,
10235                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10236                                  pf->hw.dev_caps.num_vsis,
10237                                  pf->hw.dev_caps.num_rx_qp,
10238                                  pf->hw.dev_caps.num_tx_qp);
10239                 }
10240         }
10241         if (list_type == i40e_aqc_opc_list_func_capabilities) {
10242 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10243                        + pf->hw.func_caps.num_vfs)
10244                 if (pf->hw.revision_id == 0 &&
10245                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10246                         dev_info(&pf->pdev->dev,
10247                                  "got num_vsis %d, setting num_vsis to %d\n",
10248                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10249                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10250                 }
10251         }
10252         return 0;
10253 }
10254
10255 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10256
10257 /**
10258  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10259  * @pf: board private structure
10260  **/
10261 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10262 {
10263         struct i40e_vsi *vsi;
10264
10265         /* quick workaround for an NVM issue that leaves a critical register
10266          * uninitialized
10267          */
10268         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10269                 static const u32 hkey[] = {
10270                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10271                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10272                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10273                         0x95b3a76d};
10274                 int i;
10275
10276                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10277                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10278         }
10279
10280         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
10281                 return;
10282
10283         /* find existing VSI and see if it needs configuring */
10284         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10285
10286         /* create a new VSI if none exists */
10287         if (!vsi) {
10288                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10289                                      pf->vsi[pf->lan_vsi]->seid, 0);
10290                 if (!vsi) {
10291                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10292                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10293                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10294                         return;
10295                 }
10296         }
10297
10298         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10299 }
10300
10301 /**
10302  * i40e_fdir_teardown - release the Flow Director resources
10303  * @pf: board private structure
10304  **/
10305 static void i40e_fdir_teardown(struct i40e_pf *pf)
10306 {
10307         struct i40e_vsi *vsi;
10308
10309         i40e_fdir_filter_exit(pf);
10310         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10311         if (vsi)
10312                 i40e_vsi_release(vsi);
10313 }
10314
10315 /**
10316  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10317  * @vsi: PF main vsi
10318  * @seid: seid of main or channel VSIs
10319  *
10320  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10321  * existed before reset
10322  **/
10323 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10324 {
10325         struct i40e_cloud_filter *cfilter;
10326         struct i40e_pf *pf = vsi->back;
10327         struct hlist_node *node;
10328         i40e_status ret;
10329
10330         /* Add cloud filters back if they exist */
10331         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10332                                   cloud_node) {
10333                 if (cfilter->seid != seid)
10334                         continue;
10335
10336                 if (cfilter->dst_port)
10337                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10338                                                                 true);
10339                 else
10340                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10341
10342                 if (ret) {
10343                         dev_dbg(&pf->pdev->dev,
10344                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
10345                                 i40e_stat_str(&pf->hw, ret),
10346                                 i40e_aq_str(&pf->hw,
10347                                             pf->hw.aq.asq_last_status));
10348                         return ret;
10349                 }
10350         }
10351         return 0;
10352 }
10353
10354 /**
10355  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10356  * @vsi: PF main vsi
10357  *
10358  * Rebuilds channel VSIs if they existed before reset
10359  **/
10360 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10361 {
10362         struct i40e_channel *ch, *ch_tmp;
10363         i40e_status ret;
10364
10365         if (list_empty(&vsi->ch_list))
10366                 return 0;
10367
10368         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10369                 if (!ch->initialized)
10370                         break;
10371                 /* Proceed with creation of channel (VMDq2) VSI */
10372                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10373                 if (ret) {
10374                         dev_info(&vsi->back->pdev->dev,
10375                                  "failed to rebuild channels using uplink_seid %u\n",
10376                                  vsi->uplink_seid);
10377                         return ret;
10378                 }
10379                 /* Reconfigure TX queues using QTX_CTL register */
10380                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10381                 if (ret) {
10382                         dev_info(&vsi->back->pdev->dev,
10383                                  "failed to configure TX rings for channel %u\n",
10384                                  ch->seid);
10385                         return ret;
10386                 }
10387                 /* update 'next_base_queue' */
10388                 vsi->next_base_queue = vsi->next_base_queue +
10389                                                         ch->num_queue_pairs;
10390                 if (ch->max_tx_rate) {
10391                         u64 credits = ch->max_tx_rate;
10392
10393                         if (i40e_set_bw_limit(vsi, ch->seid,
10394                                               ch->max_tx_rate))
10395                                 return -EINVAL;
10396
10397                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
10398                         dev_dbg(&vsi->back->pdev->dev,
10399                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10400                                 ch->max_tx_rate,
10401                                 credits,
10402                                 ch->seid);
10403                 }
10404                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10405                 if (ret) {
10406                         dev_dbg(&vsi->back->pdev->dev,
10407                                 "Failed to rebuild cloud filters for channel VSI %u\n",
10408                                 ch->seid);
10409                         return ret;
10410                 }
10411         }
10412         return 0;
10413 }
10414
10415 /**
10416  * i40e_prep_for_reset - prep for the core to reset
10417  * @pf: board private structure
10418  *
10419  * Close up the VFs and other things in prep for PF Reset.
10420   **/
10421 static void i40e_prep_for_reset(struct i40e_pf *pf)
10422 {
10423         struct i40e_hw *hw = &pf->hw;
10424         i40e_status ret = 0;
10425         u32 v;
10426
10427         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10428         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10429                 return;
10430         if (i40e_check_asq_alive(&pf->hw))
10431                 i40e_vc_notify_reset(pf);
10432
10433         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10434
10435         /* quiesce the VSIs and their queues that are not already DOWN */
10436         i40e_pf_quiesce_all_vsi(pf);
10437
10438         for (v = 0; v < pf->num_alloc_vsi; v++) {
10439                 if (pf->vsi[v])
10440                         pf->vsi[v]->seid = 0;
10441         }
10442
10443         i40e_shutdown_adminq(&pf->hw);
10444
10445         /* call shutdown HMC */
10446         if (hw->hmc.hmc_obj) {
10447                 ret = i40e_shutdown_lan_hmc(hw);
10448                 if (ret)
10449                         dev_warn(&pf->pdev->dev,
10450                                  "shutdown_lan_hmc failed: %d\n", ret);
10451         }
10452
10453         /* Save the current PTP time so that we can restore the time after the
10454          * reset completes.
10455          */
10456         i40e_ptp_save_hw_time(pf);
10457 }
10458
10459 /**
10460  * i40e_send_version - update firmware with driver version
10461  * @pf: PF struct
10462  */
10463 static void i40e_send_version(struct i40e_pf *pf)
10464 {
10465         struct i40e_driver_version dv;
10466
10467         dv.major_version = 0xff;
10468         dv.minor_version = 0xff;
10469         dv.build_version = 0xff;
10470         dv.subbuild_version = 0;
10471         strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10472         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10473 }
10474
10475 /**
10476  * i40e_get_oem_version - get OEM specific version information
10477  * @hw: pointer to the hardware structure
10478  **/
10479 static void i40e_get_oem_version(struct i40e_hw *hw)
10480 {
10481         u16 block_offset = 0xffff;
10482         u16 block_length = 0;
10483         u16 capabilities = 0;
10484         u16 gen_snap = 0;
10485         u16 release = 0;
10486
10487 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
10488 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
10489 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
10490 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
10491 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
10492 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
10493 #define I40E_NVM_OEM_LENGTH                     3
10494
10495         /* Check if pointer to OEM version block is valid. */
10496         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10497         if (block_offset == 0xffff)
10498                 return;
10499
10500         /* Check if OEM version block has correct length. */
10501         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10502                            &block_length);
10503         if (block_length < I40E_NVM_OEM_LENGTH)
10504                 return;
10505
10506         /* Check if OEM version format is as expected. */
10507         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10508                            &capabilities);
10509         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10510                 return;
10511
10512         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10513                            &gen_snap);
10514         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10515                            &release);
10516         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10517         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10518 }
10519
10520 /**
10521  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10522  * @pf: board private structure
10523  **/
10524 static int i40e_reset(struct i40e_pf *pf)
10525 {
10526         struct i40e_hw *hw = &pf->hw;
10527         i40e_status ret;
10528
10529         ret = i40e_pf_reset(hw);
10530         if (ret) {
10531                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10532                 set_bit(__I40E_RESET_FAILED, pf->state);
10533                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10534         } else {
10535                 pf->pfr_count++;
10536         }
10537         return ret;
10538 }
10539
10540 /**
10541  * i40e_rebuild - rebuild using a saved config
10542  * @pf: board private structure
10543  * @reinit: if the Main VSI needs to re-initialized.
10544  * @lock_acquired: indicates whether or not the lock has been acquired
10545  * before this function was called.
10546  **/
10547 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10548 {
10549         int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state);
10550         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10551         struct i40e_hw *hw = &pf->hw;
10552         i40e_status ret;
10553         u32 val;
10554         int v;
10555
10556         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10557             i40e_check_recovery_mode(pf)) {
10558                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10559         }
10560
10561         if (test_bit(__I40E_DOWN, pf->state) &&
10562             !test_bit(__I40E_RECOVERY_MODE, pf->state) &&
10563             !old_recovery_mode_bit)
10564                 goto clear_recovery;
10565         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10566
10567         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10568         ret = i40e_init_adminq(&pf->hw);
10569         if (ret) {
10570                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
10571                          i40e_stat_str(&pf->hw, ret),
10572                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10573                 goto clear_recovery;
10574         }
10575         i40e_get_oem_version(&pf->hw);
10576
10577         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10578             ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
10579              hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
10580                 /* The following delay is necessary for 4.33 firmware and older
10581                  * to recover after EMP reset. 200 ms should suffice but we
10582                  * put here 300 ms to be sure that FW is ready to operate
10583                  * after reset.
10584                  */
10585                 mdelay(300);
10586         }
10587
10588         /* re-verify the eeprom if we just had an EMP reset */
10589         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10590                 i40e_verify_eeprom(pf);
10591
10592         /* if we are going out of or into recovery mode we have to act
10593          * accordingly with regard to resources initialization
10594          * and deinitialization
10595          */
10596         if (test_bit(__I40E_RECOVERY_MODE, pf->state) ||
10597             old_recovery_mode_bit) {
10598                 if (i40e_get_capabilities(pf,
10599                                           i40e_aqc_opc_list_func_capabilities))
10600                         goto end_unlock;
10601
10602                 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10603                         /* we're staying in recovery mode so we'll reinitialize
10604                          * misc vector here
10605                          */
10606                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10607                                 goto end_unlock;
10608                 } else {
10609                         if (!lock_acquired)
10610                                 rtnl_lock();
10611                         /* we're going out of recovery mode so we'll free
10612                          * the IRQ allocated specifically for recovery mode
10613                          * and restore the interrupt scheme
10614                          */
10615                         free_irq(pf->pdev->irq, pf);
10616                         i40e_clear_interrupt_scheme(pf);
10617                         if (i40e_restore_interrupt_scheme(pf))
10618                                 goto end_unlock;
10619                 }
10620
10621                 /* tell the firmware that we're starting */
10622                 i40e_send_version(pf);
10623
10624                 /* bail out in case recovery mode was detected, as there is
10625                  * no need for further configuration.
10626                  */
10627                 goto end_unlock;
10628         }
10629
10630         i40e_clear_pxe_mode(hw);
10631         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10632         if (ret)
10633                 goto end_core_reset;
10634
10635         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10636                                 hw->func_caps.num_rx_qp, 0, 0);
10637         if (ret) {
10638                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10639                 goto end_core_reset;
10640         }
10641         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10642         if (ret) {
10643                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10644                 goto end_core_reset;
10645         }
10646
10647 #ifdef CONFIG_I40E_DCB
10648         /* Enable FW to write a default DCB config on link-up
10649          * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10650          * is not supported with new link speed
10651          */
10652         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
10653                 i40e_aq_set_dcb_parameters(hw, false, NULL);
10654         } else {
10655                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10656                     (hw->phy.link_info.link_speed &
10657                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10658                         i40e_aq_set_dcb_parameters(hw, false, NULL);
10659                         dev_warn(&pf->pdev->dev,
10660                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10661                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10662                 } else {
10663                         i40e_aq_set_dcb_parameters(hw, true, NULL);
10664                         ret = i40e_init_pf_dcb(pf);
10665                         if (ret) {
10666                                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10667                                          ret);
10668                                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10669                                 /* Continue without DCB enabled */
10670                         }
10671                 }
10672         }
10673
10674 #endif /* CONFIG_I40E_DCB */
10675         if (!lock_acquired)
10676                 rtnl_lock();
10677         ret = i40e_setup_pf_switch(pf, reinit, true);
10678         if (ret)
10679                 goto end_unlock;
10680
10681         /* The driver only wants link up/down and module qualification
10682          * reports from firmware.  Note the negative logic.
10683          */
10684         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10685                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10686                                          I40E_AQ_EVENT_MEDIA_NA |
10687                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10688         if (ret)
10689                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10690                          i40e_stat_str(&pf->hw, ret),
10691                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10692
10693         /* Rebuild the VSIs and VEBs that existed before reset.
10694          * They are still in our local switch element arrays, so only
10695          * need to rebuild the switch model in the HW.
10696          *
10697          * If there were VEBs but the reconstitution failed, we'll try
10698          * to recover minimal use by getting the basic PF VSI working.
10699          */
10700         if (vsi->uplink_seid != pf->mac_seid) {
10701                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10702                 /* find the one VEB connected to the MAC, and find orphans */
10703                 for (v = 0; v < I40E_MAX_VEB; v++) {
10704                         if (!pf->veb[v])
10705                                 continue;
10706
10707                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10708                             pf->veb[v]->uplink_seid == 0) {
10709                                 ret = i40e_reconstitute_veb(pf->veb[v]);
10710
10711                                 if (!ret)
10712                                         continue;
10713
10714                                 /* If Main VEB failed, we're in deep doodoo,
10715                                  * so give up rebuilding the switch and set up
10716                                  * for minimal rebuild of PF VSI.
10717                                  * If orphan failed, we'll report the error
10718                                  * but try to keep going.
10719                                  */
10720                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10721                                         dev_info(&pf->pdev->dev,
10722                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10723                                                  ret);
10724                                         vsi->uplink_seid = pf->mac_seid;
10725                                         break;
10726                                 } else if (pf->veb[v]->uplink_seid == 0) {
10727                                         dev_info(&pf->pdev->dev,
10728                                                  "rebuild of orphan VEB failed: %d\n",
10729                                                  ret);
10730                                 }
10731                         }
10732                 }
10733         }
10734
10735         if (vsi->uplink_seid == pf->mac_seid) {
10736                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10737                 /* no VEB, so rebuild only the Main VSI */
10738                 ret = i40e_add_vsi(vsi);
10739                 if (ret) {
10740                         dev_info(&pf->pdev->dev,
10741                                  "rebuild of Main VSI failed: %d\n", ret);
10742                         goto end_unlock;
10743                 }
10744         }
10745
10746         if (vsi->mqprio_qopt.max_rate[0]) {
10747                 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
10748                 u64 credits = 0;
10749
10750                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
10751                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10752                 if (ret)
10753                         goto end_unlock;
10754
10755                 credits = max_tx_rate;
10756                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10757                 dev_dbg(&vsi->back->pdev->dev,
10758                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10759                         max_tx_rate,
10760                         credits,
10761                         vsi->seid);
10762         }
10763
10764         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10765         if (ret)
10766                 goto end_unlock;
10767
10768         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
10769          * for this main VSI if they exist
10770          */
10771         ret = i40e_rebuild_channels(vsi);
10772         if (ret)
10773                 goto end_unlock;
10774
10775         /* Reconfigure hardware for allowing smaller MSS in the case
10776          * of TSO, so that we avoid the MDD being fired and causing
10777          * a reset in the case of small MSS+TSO.
10778          */
10779 #define I40E_REG_MSS          0x000E64DC
10780 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
10781 #define I40E_64BYTE_MSS       0x400000
10782         val = rd32(hw, I40E_REG_MSS);
10783         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10784                 val &= ~I40E_REG_MSS_MIN_MASK;
10785                 val |= I40E_64BYTE_MSS;
10786                 wr32(hw, I40E_REG_MSS, val);
10787         }
10788
10789         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10790                 msleep(75);
10791                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10792                 if (ret)
10793                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10794                                  i40e_stat_str(&pf->hw, ret),
10795                                  i40e_aq_str(&pf->hw,
10796                                              pf->hw.aq.asq_last_status));
10797         }
10798         /* reinit the misc interrupt */
10799         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10800                 ret = i40e_setup_misc_vector(pf);
10801
10802         /* Add a filter to drop all Flow control frames from any VSI from being
10803          * transmitted. By doing so we stop a malicious VF from sending out
10804          * PAUSE or PFC frames and potentially controlling traffic for other
10805          * PF/VF VSIs.
10806          * The FW can still send Flow control frames if enabled.
10807          */
10808         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10809                                                        pf->main_vsi_seid);
10810
10811         /* restart the VSIs that were rebuilt and running before the reset */
10812         i40e_pf_unquiesce_all_vsi(pf);
10813
10814         /* Release the RTNL lock before we start resetting VFs */
10815         if (!lock_acquired)
10816                 rtnl_unlock();
10817
10818         /* Restore promiscuous settings */
10819         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10820         if (ret)
10821                 dev_warn(&pf->pdev->dev,
10822                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10823                          pf->cur_promisc ? "on" : "off",
10824                          i40e_stat_str(&pf->hw, ret),
10825                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10826
10827         i40e_reset_all_vfs(pf, true);
10828
10829         /* tell the firmware that we're starting */
10830         i40e_send_version(pf);
10831
10832         /* We've already released the lock, so don't do it again */
10833         goto end_core_reset;
10834
10835 end_unlock:
10836         if (!lock_acquired)
10837                 rtnl_unlock();
10838 end_core_reset:
10839         clear_bit(__I40E_RESET_FAILED, pf->state);
10840 clear_recovery:
10841         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10842         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10843 }
10844
10845 /**
10846  * i40e_reset_and_rebuild - reset and rebuild using a saved config
10847  * @pf: board private structure
10848  * @reinit: if the Main VSI needs to re-initialized.
10849  * @lock_acquired: indicates whether or not the lock has been acquired
10850  * before this function was called.
10851  **/
10852 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10853                                    bool lock_acquired)
10854 {
10855         int ret;
10856         /* Now we wait for GRST to settle out.
10857          * We don't have to delete the VEBs or VSIs from the hw switch
10858          * because the reset will make them disappear.
10859          */
10860         ret = i40e_reset(pf);
10861         if (!ret)
10862                 i40e_rebuild(pf, reinit, lock_acquired);
10863 }
10864
10865 /**
10866  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
10867  * @pf: board private structure
10868  *
10869  * Close up the VFs and other things in prep for a Core Reset,
10870  * then get ready to rebuild the world.
10871  * @lock_acquired: indicates whether or not the lock has been acquired
10872  * before this function was called.
10873  **/
10874 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10875 {
10876         i40e_prep_for_reset(pf);
10877         i40e_reset_and_rebuild(pf, false, lock_acquired);
10878 }
10879
10880 /**
10881  * i40e_handle_mdd_event
10882  * @pf: pointer to the PF structure
10883  *
10884  * Called from the MDD irq handler to identify possibly malicious vfs
10885  **/
10886 static void i40e_handle_mdd_event(struct i40e_pf *pf)
10887 {
10888         struct i40e_hw *hw = &pf->hw;
10889         bool mdd_detected = false;
10890         struct i40e_vf *vf;
10891         u32 reg;
10892         int i;
10893
10894         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10895                 return;
10896
10897         /* find what triggered the MDD event */
10898         reg = rd32(hw, I40E_GL_MDET_TX);
10899         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10900                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10901                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10902                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10903                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10904                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10905                                 I40E_GL_MDET_TX_EVENT_SHIFT;
10906                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10907                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10908                                 pf->hw.func_caps.base_queue;
10909                 if (netif_msg_tx_err(pf))
10910                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10911                                  event, queue, pf_num, vf_num);
10912                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10913                 mdd_detected = true;
10914         }
10915         reg = rd32(hw, I40E_GL_MDET_RX);
10916         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10917                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10918                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10919                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10920                                 I40E_GL_MDET_RX_EVENT_SHIFT;
10921                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10922                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10923                                 pf->hw.func_caps.base_queue;
10924                 if (netif_msg_rx_err(pf))
10925                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10926                                  event, queue, func);
10927                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10928                 mdd_detected = true;
10929         }
10930
10931         if (mdd_detected) {
10932                 reg = rd32(hw, I40E_PF_MDET_TX);
10933                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10934                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10935                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10936                 }
10937                 reg = rd32(hw, I40E_PF_MDET_RX);
10938                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10939                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10940                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10941                 }
10942         }
10943
10944         /* see if one of the VFs needs its hand slapped */
10945         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10946                 vf = &(pf->vf[i]);
10947                 reg = rd32(hw, I40E_VP_MDET_TX(i));
10948                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10949                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10950                         vf->num_mdd_events++;
10951                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10952                                  i);
10953                         dev_info(&pf->pdev->dev,
10954                                  "Use PF Control I/F to re-enable the VF\n");
10955                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10956                 }
10957
10958                 reg = rd32(hw, I40E_VP_MDET_RX(i));
10959                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10960                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10961                         vf->num_mdd_events++;
10962                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10963                                  i);
10964                         dev_info(&pf->pdev->dev,
10965                                  "Use PF Control I/F to re-enable the VF\n");
10966                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10967                 }
10968         }
10969
10970         /* re-enable mdd interrupt cause */
10971         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10972         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10973         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10974         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10975         i40e_flush(hw);
10976 }
10977
10978 /**
10979  * i40e_service_task - Run the driver's async subtasks
10980  * @work: pointer to work_struct containing our data
10981  **/
10982 static void i40e_service_task(struct work_struct *work)
10983 {
10984         struct i40e_pf *pf = container_of(work,
10985                                           struct i40e_pf,
10986                                           service_task);
10987         unsigned long start_time = jiffies;
10988
10989         /* don't bother with service tasks if a reset is in progress */
10990         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10991             test_bit(__I40E_SUSPENDED, pf->state))
10992                 return;
10993
10994         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10995                 return;
10996
10997         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10998                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10999                 i40e_sync_filters_subtask(pf);
11000                 i40e_reset_subtask(pf);
11001                 i40e_handle_mdd_event(pf);
11002                 i40e_vc_process_vflr_event(pf);
11003                 i40e_watchdog_subtask(pf);
11004                 i40e_fdir_reinit_subtask(pf);
11005                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11006                         /* Client subtask will reopen next time through. */
11007                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
11008                                                            true);
11009                 } else {
11010                         i40e_client_subtask(pf);
11011                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11012                                                pf->state))
11013                                 i40e_notify_client_of_l2_param_changes(
11014                                                                 pf->vsi[pf->lan_vsi]);
11015                 }
11016                 i40e_sync_filters_subtask(pf);
11017         } else {
11018                 i40e_reset_subtask(pf);
11019         }
11020
11021         i40e_clean_adminq_subtask(pf);
11022
11023         /* flush memory to make sure state is correct before next watchdog */
11024         smp_mb__before_atomic();
11025         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11026
11027         /* If the tasks have taken longer than one timer cycle or there
11028          * is more work to be done, reschedule the service task now
11029          * rather than wait for the timer to tick again.
11030          */
11031         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11032             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
11033             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
11034             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11035                 i40e_service_event_schedule(pf);
11036 }
11037
11038 /**
11039  * i40e_service_timer - timer callback
11040  * @t: timer list pointer
11041  **/
11042 static void i40e_service_timer(struct timer_list *t)
11043 {
11044         struct i40e_pf *pf = from_timer(pf, t, service_timer);
11045
11046         mod_timer(&pf->service_timer,
11047                   round_jiffies(jiffies + pf->service_timer_period));
11048         i40e_service_event_schedule(pf);
11049 }
11050
11051 /**
11052  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11053  * @vsi: the VSI being configured
11054  **/
11055 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11056 {
11057         struct i40e_pf *pf = vsi->back;
11058
11059         switch (vsi->type) {
11060         case I40E_VSI_MAIN:
11061                 vsi->alloc_queue_pairs = pf->num_lan_qps;
11062                 if (!vsi->num_tx_desc)
11063                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11064                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11065                 if (!vsi->num_rx_desc)
11066                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11067                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11068                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11069                         vsi->num_q_vectors = pf->num_lan_msix;
11070                 else
11071                         vsi->num_q_vectors = 1;
11072
11073                 break;
11074
11075         case I40E_VSI_FDIR:
11076                 vsi->alloc_queue_pairs = 1;
11077                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11078                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11079                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11080                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11081                 vsi->num_q_vectors = pf->num_fdsb_msix;
11082                 break;
11083
11084         case I40E_VSI_VMDQ2:
11085                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11086                 if (!vsi->num_tx_desc)
11087                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11088                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11089                 if (!vsi->num_rx_desc)
11090                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11091                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11092                 vsi->num_q_vectors = pf->num_vmdq_msix;
11093                 break;
11094
11095         case I40E_VSI_SRIOV:
11096                 vsi->alloc_queue_pairs = pf->num_vf_qps;
11097                 if (!vsi->num_tx_desc)
11098                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11099                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11100                 if (!vsi->num_rx_desc)
11101                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11102                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11103                 break;
11104
11105         default:
11106                 WARN_ON(1);
11107                 return -ENODATA;
11108         }
11109
11110         if (is_kdump_kernel()) {
11111                 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11112                 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11113         }
11114
11115         return 0;
11116 }
11117
11118 /**
11119  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11120  * @vsi: VSI pointer
11121  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11122  *
11123  * On error: returns error code (negative)
11124  * On success: returns 0
11125  **/
11126 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11127 {
11128         struct i40e_ring **next_rings;
11129         int size;
11130         int ret = 0;
11131
11132         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11133         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11134                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11135         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11136         if (!vsi->tx_rings)
11137                 return -ENOMEM;
11138         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11139         if (i40e_enabled_xdp_vsi(vsi)) {
11140                 vsi->xdp_rings = next_rings;
11141                 next_rings += vsi->alloc_queue_pairs;
11142         }
11143         vsi->rx_rings = next_rings;
11144
11145         if (alloc_qvectors) {
11146                 /* allocate memory for q_vector pointers */
11147                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11148                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11149                 if (!vsi->q_vectors) {
11150                         ret = -ENOMEM;
11151                         goto err_vectors;
11152                 }
11153         }
11154         return ret;
11155
11156 err_vectors:
11157         kfree(vsi->tx_rings);
11158         return ret;
11159 }
11160
11161 /**
11162  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11163  * @pf: board private structure
11164  * @type: type of VSI
11165  *
11166  * On error: returns error code (negative)
11167  * On success: returns vsi index in PF (positive)
11168  **/
11169 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11170 {
11171         int ret = -ENODEV;
11172         struct i40e_vsi *vsi;
11173         int vsi_idx;
11174         int i;
11175
11176         /* Need to protect the allocation of the VSIs at the PF level */
11177         mutex_lock(&pf->switch_mutex);
11178
11179         /* VSI list may be fragmented if VSI creation/destruction has
11180          * been happening.  We can afford to do a quick scan to look
11181          * for any free VSIs in the list.
11182          *
11183          * find next empty vsi slot, looping back around if necessary
11184          */
11185         i = pf->next_vsi;
11186         while (i < pf->num_alloc_vsi && pf->vsi[i])
11187                 i++;
11188         if (i >= pf->num_alloc_vsi) {
11189                 i = 0;
11190                 while (i < pf->next_vsi && pf->vsi[i])
11191                         i++;
11192         }
11193
11194         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11195                 vsi_idx = i;             /* Found one! */
11196         } else {
11197                 ret = -ENODEV;
11198                 goto unlock_pf;  /* out of VSI slots! */
11199         }
11200         pf->next_vsi = ++i;
11201
11202         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11203         if (!vsi) {
11204                 ret = -ENOMEM;
11205                 goto unlock_pf;
11206         }
11207         vsi->type = type;
11208         vsi->back = pf;
11209         set_bit(__I40E_VSI_DOWN, vsi->state);
11210         vsi->flags = 0;
11211         vsi->idx = vsi_idx;
11212         vsi->int_rate_limit = 0;
11213         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11214                                 pf->rss_table_size : 64;
11215         vsi->netdev_registered = false;
11216         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11217         hash_init(vsi->mac_filter_hash);
11218         vsi->irqs_ready = false;
11219
11220         if (type == I40E_VSI_MAIN) {
11221                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11222                 if (!vsi->af_xdp_zc_qps)
11223                         goto err_rings;
11224         }
11225
11226         ret = i40e_set_num_rings_in_vsi(vsi);
11227         if (ret)
11228                 goto err_rings;
11229
11230         ret = i40e_vsi_alloc_arrays(vsi, true);
11231         if (ret)
11232                 goto err_rings;
11233
11234         /* Setup default MSIX irq handler for VSI */
11235         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11236
11237         /* Initialize VSI lock */
11238         spin_lock_init(&vsi->mac_filter_hash_lock);
11239         pf->vsi[vsi_idx] = vsi;
11240         ret = vsi_idx;
11241         goto unlock_pf;
11242
11243 err_rings:
11244         bitmap_free(vsi->af_xdp_zc_qps);
11245         pf->next_vsi = i - 1;
11246         kfree(vsi);
11247 unlock_pf:
11248         mutex_unlock(&pf->switch_mutex);
11249         return ret;
11250 }
11251
11252 /**
11253  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11254  * @vsi: VSI pointer
11255  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11256  *
11257  * On error: returns error code (negative)
11258  * On success: returns 0
11259  **/
11260 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11261 {
11262         /* free the ring and vector containers */
11263         if (free_qvectors) {
11264                 kfree(vsi->q_vectors);
11265                 vsi->q_vectors = NULL;
11266         }
11267         kfree(vsi->tx_rings);
11268         vsi->tx_rings = NULL;
11269         vsi->rx_rings = NULL;
11270         vsi->xdp_rings = NULL;
11271 }
11272
11273 /**
11274  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11275  * and lookup table
11276  * @vsi: Pointer to VSI structure
11277  */
11278 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11279 {
11280         if (!vsi)
11281                 return;
11282
11283         kfree(vsi->rss_hkey_user);
11284         vsi->rss_hkey_user = NULL;
11285
11286         kfree(vsi->rss_lut_user);
11287         vsi->rss_lut_user = NULL;
11288 }
11289
11290 /**
11291  * i40e_vsi_clear - Deallocate the VSI provided
11292  * @vsi: the VSI being un-configured
11293  **/
11294 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11295 {
11296         struct i40e_pf *pf;
11297
11298         if (!vsi)
11299                 return 0;
11300
11301         if (!vsi->back)
11302                 goto free_vsi;
11303         pf = vsi->back;
11304
11305         mutex_lock(&pf->switch_mutex);
11306         if (!pf->vsi[vsi->idx]) {
11307                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11308                         vsi->idx, vsi->idx, vsi->type);
11309                 goto unlock_vsi;
11310         }
11311
11312         if (pf->vsi[vsi->idx] != vsi) {
11313                 dev_err(&pf->pdev->dev,
11314                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11315                         pf->vsi[vsi->idx]->idx,
11316                         pf->vsi[vsi->idx]->type,
11317                         vsi->idx, vsi->type);
11318                 goto unlock_vsi;
11319         }
11320
11321         /* updates the PF for this cleared vsi */
11322         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11323         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11324
11325         bitmap_free(vsi->af_xdp_zc_qps);
11326         i40e_vsi_free_arrays(vsi, true);
11327         i40e_clear_rss_config_user(vsi);
11328
11329         pf->vsi[vsi->idx] = NULL;
11330         if (vsi->idx < pf->next_vsi)
11331                 pf->next_vsi = vsi->idx;
11332
11333 unlock_vsi:
11334         mutex_unlock(&pf->switch_mutex);
11335 free_vsi:
11336         kfree(vsi);
11337
11338         return 0;
11339 }
11340
11341 /**
11342  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11343  * @vsi: the VSI being cleaned
11344  **/
11345 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11346 {
11347         int i;
11348
11349         if (vsi->tx_rings && vsi->tx_rings[0]) {
11350                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11351                         kfree_rcu(vsi->tx_rings[i], rcu);
11352                         WRITE_ONCE(vsi->tx_rings[i], NULL);
11353                         WRITE_ONCE(vsi->rx_rings[i], NULL);
11354                         if (vsi->xdp_rings)
11355                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11356                 }
11357         }
11358 }
11359
11360 /**
11361  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11362  * @vsi: the VSI being configured
11363  **/
11364 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11365 {
11366         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11367         struct i40e_pf *pf = vsi->back;
11368         struct i40e_ring *ring;
11369
11370         /* Set basic values in the rings to be used later during open() */
11371         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11372                 /* allocate space for both Tx and Rx in one shot */
11373                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11374                 if (!ring)
11375                         goto err_out;
11376
11377                 ring->queue_index = i;
11378                 ring->reg_idx = vsi->base_queue + i;
11379                 ring->ring_active = false;
11380                 ring->vsi = vsi;
11381                 ring->netdev = vsi->netdev;
11382                 ring->dev = &pf->pdev->dev;
11383                 ring->count = vsi->num_tx_desc;
11384                 ring->size = 0;
11385                 ring->dcb_tc = 0;
11386                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11387                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11388                 ring->itr_setting = pf->tx_itr_default;
11389                 WRITE_ONCE(vsi->tx_rings[i], ring++);
11390
11391                 if (!i40e_enabled_xdp_vsi(vsi))
11392                         goto setup_rx;
11393
11394                 ring->queue_index = vsi->alloc_queue_pairs + i;
11395                 ring->reg_idx = vsi->base_queue + ring->queue_index;
11396                 ring->ring_active = false;
11397                 ring->vsi = vsi;
11398                 ring->netdev = NULL;
11399                 ring->dev = &pf->pdev->dev;
11400                 ring->count = vsi->num_tx_desc;
11401                 ring->size = 0;
11402                 ring->dcb_tc = 0;
11403                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11404                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11405                 set_ring_xdp(ring);
11406                 ring->itr_setting = pf->tx_itr_default;
11407                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11408
11409 setup_rx:
11410                 ring->queue_index = i;
11411                 ring->reg_idx = vsi->base_queue + i;
11412                 ring->ring_active = false;
11413                 ring->vsi = vsi;
11414                 ring->netdev = vsi->netdev;
11415                 ring->dev = &pf->pdev->dev;
11416                 ring->count = vsi->num_rx_desc;
11417                 ring->size = 0;
11418                 ring->dcb_tc = 0;
11419                 ring->itr_setting = pf->rx_itr_default;
11420                 WRITE_ONCE(vsi->rx_rings[i], ring);
11421         }
11422
11423         return 0;
11424
11425 err_out:
11426         i40e_vsi_clear_rings(vsi);
11427         return -ENOMEM;
11428 }
11429
11430 /**
11431  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11432  * @pf: board private structure
11433  * @vectors: the number of MSI-X vectors to request
11434  *
11435  * Returns the number of vectors reserved, or error
11436  **/
11437 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11438 {
11439         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11440                                         I40E_MIN_MSIX, vectors);
11441         if (vectors < 0) {
11442                 dev_info(&pf->pdev->dev,
11443                          "MSI-X vector reservation failed: %d\n", vectors);
11444                 vectors = 0;
11445         }
11446
11447         return vectors;
11448 }
11449
11450 /**
11451  * i40e_init_msix - Setup the MSIX capability
11452  * @pf: board private structure
11453  *
11454  * Work with the OS to set up the MSIX vectors needed.
11455  *
11456  * Returns the number of vectors reserved or negative on failure
11457  **/
11458 static int i40e_init_msix(struct i40e_pf *pf)
11459 {
11460         struct i40e_hw *hw = &pf->hw;
11461         int cpus, extra_vectors;
11462         int vectors_left;
11463         int v_budget, i;
11464         int v_actual;
11465         int iwarp_requested = 0;
11466
11467         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11468                 return -ENODEV;
11469
11470         /* The number of vectors we'll request will be comprised of:
11471          *   - Add 1 for "other" cause for Admin Queue events, etc.
11472          *   - The number of LAN queue pairs
11473          *      - Queues being used for RSS.
11474          *              We don't need as many as max_rss_size vectors.
11475          *              use rss_size instead in the calculation since that
11476          *              is governed by number of cpus in the system.
11477          *      - assumes symmetric Tx/Rx pairing
11478          *   - The number of VMDq pairs
11479          *   - The CPU count within the NUMA node if iWARP is enabled
11480          * Once we count this up, try the request.
11481          *
11482          * If we can't get what we want, we'll simplify to nearly nothing
11483          * and try again.  If that still fails, we punt.
11484          */
11485         vectors_left = hw->func_caps.num_msix_vectors;
11486         v_budget = 0;
11487
11488         /* reserve one vector for miscellaneous handler */
11489         if (vectors_left) {
11490                 v_budget++;
11491                 vectors_left--;
11492         }
11493
11494         /* reserve some vectors for the main PF traffic queues. Initially we
11495          * only reserve at most 50% of the available vectors, in the case that
11496          * the number of online CPUs is large. This ensures that we can enable
11497          * extra features as well. Once we've enabled the other features, we
11498          * will use any remaining vectors to reach as close as we can to the
11499          * number of online CPUs.
11500          */
11501         cpus = num_online_cpus();
11502         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11503         vectors_left -= pf->num_lan_msix;
11504
11505         /* reserve one vector for sideband flow director */
11506         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11507                 if (vectors_left) {
11508                         pf->num_fdsb_msix = 1;
11509                         v_budget++;
11510                         vectors_left--;
11511                 } else {
11512                         pf->num_fdsb_msix = 0;
11513                 }
11514         }
11515
11516         /* can we reserve enough for iWARP? */
11517         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11518                 iwarp_requested = pf->num_iwarp_msix;
11519
11520                 if (!vectors_left)
11521                         pf->num_iwarp_msix = 0;
11522                 else if (vectors_left < pf->num_iwarp_msix)
11523                         pf->num_iwarp_msix = 1;
11524                 v_budget += pf->num_iwarp_msix;
11525                 vectors_left -= pf->num_iwarp_msix;
11526         }
11527
11528         /* any vectors left over go for VMDq support */
11529         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11530                 if (!vectors_left) {
11531                         pf->num_vmdq_msix = 0;
11532                         pf->num_vmdq_qps = 0;
11533                 } else {
11534                         int vmdq_vecs_wanted =
11535                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11536                         int vmdq_vecs =
11537                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11538
11539                         /* if we're short on vectors for what's desired, we limit
11540                          * the queues per vmdq.  If this is still more than are
11541                          * available, the user will need to change the number of
11542                          * queues/vectors used by the PF later with the ethtool
11543                          * channels command
11544                          */
11545                         if (vectors_left < vmdq_vecs_wanted) {
11546                                 pf->num_vmdq_qps = 1;
11547                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11548                                 vmdq_vecs = min_t(int,
11549                                                   vectors_left,
11550                                                   vmdq_vecs_wanted);
11551                         }
11552                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11553
11554                         v_budget += vmdq_vecs;
11555                         vectors_left -= vmdq_vecs;
11556                 }
11557         }
11558
11559         /* On systems with a large number of SMP cores, we previously limited
11560          * the number of vectors for num_lan_msix to be at most 50% of the
11561          * available vectors, to allow for other features. Now, we add back
11562          * the remaining vectors. However, we ensure that the total
11563          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11564          * calculate the number of vectors we can add without going over the
11565          * cap of CPUs. For systems with a small number of CPUs this will be
11566          * zero.
11567          */
11568         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11569         pf->num_lan_msix += extra_vectors;
11570         vectors_left -= extra_vectors;
11571
11572         WARN(vectors_left < 0,
11573              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11574
11575         v_budget += pf->num_lan_msix;
11576         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11577                                    GFP_KERNEL);
11578         if (!pf->msix_entries)
11579                 return -ENOMEM;
11580
11581         for (i = 0; i < v_budget; i++)
11582                 pf->msix_entries[i].entry = i;
11583         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11584
11585         if (v_actual < I40E_MIN_MSIX) {
11586                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11587                 kfree(pf->msix_entries);
11588                 pf->msix_entries = NULL;
11589                 pci_disable_msix(pf->pdev);
11590                 return -ENODEV;
11591
11592         } else if (v_actual == I40E_MIN_MSIX) {
11593                 /* Adjust for minimal MSIX use */
11594                 pf->num_vmdq_vsis = 0;
11595                 pf->num_vmdq_qps = 0;
11596                 pf->num_lan_qps = 1;
11597                 pf->num_lan_msix = 1;
11598
11599         } else if (v_actual != v_budget) {
11600                 /* If we have limited resources, we will start with no vectors
11601                  * for the special features and then allocate vectors to some
11602                  * of these features based on the policy and at the end disable
11603                  * the features that did not get any vectors.
11604                  */
11605                 int vec;
11606
11607                 dev_info(&pf->pdev->dev,
11608                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11609                          v_actual, v_budget);
11610                 /* reserve the misc vector */
11611                 vec = v_actual - 1;
11612
11613                 /* Scale vector usage down */
11614                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11615                 pf->num_vmdq_vsis = 1;
11616                 pf->num_vmdq_qps = 1;
11617
11618                 /* partition out the remaining vectors */
11619                 switch (vec) {
11620                 case 2:
11621                         pf->num_lan_msix = 1;
11622                         break;
11623                 case 3:
11624                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11625                                 pf->num_lan_msix = 1;
11626                                 pf->num_iwarp_msix = 1;
11627                         } else {
11628                                 pf->num_lan_msix = 2;
11629                         }
11630                         break;
11631                 default:
11632                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11633                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11634                                                  iwarp_requested);
11635                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11636                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11637                         } else {
11638                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11639                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11640                         }
11641                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11642                                 pf->num_fdsb_msix = 1;
11643                                 vec--;
11644                         }
11645                         pf->num_lan_msix = min_t(int,
11646                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11647                                                               pf->num_lan_msix);
11648                         pf->num_lan_qps = pf->num_lan_msix;
11649                         break;
11650                 }
11651         }
11652
11653         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11654             (pf->num_fdsb_msix == 0)) {
11655                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11656                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11657                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11658         }
11659         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11660             (pf->num_vmdq_msix == 0)) {
11661                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11662                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11663         }
11664
11665         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11666             (pf->num_iwarp_msix == 0)) {
11667                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11668                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11669         }
11670         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11671                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11672                    pf->num_lan_msix,
11673                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11674                    pf->num_fdsb_msix,
11675                    pf->num_iwarp_msix);
11676
11677         return v_actual;
11678 }
11679
11680 /**
11681  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11682  * @vsi: the VSI being configured
11683  * @v_idx: index of the vector in the vsi struct
11684  *
11685  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11686  **/
11687 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11688 {
11689         struct i40e_q_vector *q_vector;
11690
11691         /* allocate q_vector */
11692         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11693         if (!q_vector)
11694                 return -ENOMEM;
11695
11696         q_vector->vsi = vsi;
11697         q_vector->v_idx = v_idx;
11698         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11699
11700         if (vsi->netdev)
11701                 netif_napi_add(vsi->netdev, &q_vector->napi,
11702                                i40e_napi_poll, NAPI_POLL_WEIGHT);
11703
11704         /* tie q_vector and vsi together */
11705         vsi->q_vectors[v_idx] = q_vector;
11706
11707         return 0;
11708 }
11709
11710 /**
11711  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11712  * @vsi: the VSI being configured
11713  *
11714  * We allocate one q_vector per queue interrupt.  If allocation fails we
11715  * return -ENOMEM.
11716  **/
11717 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11718 {
11719         struct i40e_pf *pf = vsi->back;
11720         int err, v_idx, num_q_vectors;
11721
11722         /* if not MSIX, give the one vector only to the LAN VSI */
11723         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11724                 num_q_vectors = vsi->num_q_vectors;
11725         else if (vsi == pf->vsi[pf->lan_vsi])
11726                 num_q_vectors = 1;
11727         else
11728                 return -EINVAL;
11729
11730         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11731                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
11732                 if (err)
11733                         goto err_out;
11734         }
11735
11736         return 0;
11737
11738 err_out:
11739         while (v_idx--)
11740                 i40e_free_q_vector(vsi, v_idx);
11741
11742         return err;
11743 }
11744
11745 /**
11746  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
11747  * @pf: board private structure to initialize
11748  **/
11749 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11750 {
11751         int vectors = 0;
11752         ssize_t size;
11753
11754         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11755                 vectors = i40e_init_msix(pf);
11756                 if (vectors < 0) {
11757                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
11758                                        I40E_FLAG_IWARP_ENABLED  |
11759                                        I40E_FLAG_RSS_ENABLED    |
11760                                        I40E_FLAG_DCB_CAPABLE    |
11761                                        I40E_FLAG_DCB_ENABLED    |
11762                                        I40E_FLAG_SRIOV_ENABLED  |
11763                                        I40E_FLAG_FD_SB_ENABLED  |
11764                                        I40E_FLAG_FD_ATR_ENABLED |
11765                                        I40E_FLAG_VMDQ_ENABLED);
11766                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11767
11768                         /* rework the queue expectations without MSIX */
11769                         i40e_determine_queue_usage(pf);
11770                 }
11771         }
11772
11773         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11774             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11775                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11776                 vectors = pci_enable_msi(pf->pdev);
11777                 if (vectors < 0) {
11778                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11779                                  vectors);
11780                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11781                 }
11782                 vectors = 1;  /* one MSI or Legacy vector */
11783         }
11784
11785         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11786                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11787
11788         /* set up vector assignment tracking */
11789         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11790         pf->irq_pile = kzalloc(size, GFP_KERNEL);
11791         if (!pf->irq_pile)
11792                 return -ENOMEM;
11793
11794         pf->irq_pile->num_entries = vectors;
11795         pf->irq_pile->search_hint = 0;
11796
11797         /* track first vector for misc interrupts, ignore return */
11798         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11799
11800         return 0;
11801 }
11802
11803 /**
11804  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
11805  * @pf: private board data structure
11806  *
11807  * Restore the interrupt scheme that was cleared when we suspended the
11808  * device. This should be called during resume to re-allocate the q_vectors
11809  * and reacquire IRQs.
11810  */
11811 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11812 {
11813         int err, i;
11814
11815         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
11816          * scheme. We need to re-enabled them here in order to attempt to
11817          * re-acquire the MSI or MSI-X vectors
11818          */
11819         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11820
11821         err = i40e_init_interrupt_scheme(pf);
11822         if (err)
11823                 return err;
11824
11825         /* Now that we've re-acquired IRQs, we need to remap the vectors and
11826          * rings together again.
11827          */
11828         for (i = 0; i < pf->num_alloc_vsi; i++) {
11829                 if (pf->vsi[i]) {
11830                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11831                         if (err)
11832                                 goto err_unwind;
11833                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11834                 }
11835         }
11836
11837         err = i40e_setup_misc_vector(pf);
11838         if (err)
11839                 goto err_unwind;
11840
11841         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11842                 i40e_client_update_msix_info(pf);
11843
11844         return 0;
11845
11846 err_unwind:
11847         while (i--) {
11848                 if (pf->vsi[i])
11849                         i40e_vsi_free_q_vectors(pf->vsi[i]);
11850         }
11851
11852         return err;
11853 }
11854
11855 /**
11856  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
11857  * non queue events in recovery mode
11858  * @pf: board private structure
11859  *
11860  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
11861  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
11862  * This is handled differently than in recovery mode since no Tx/Rx resources
11863  * are being allocated.
11864  **/
11865 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11866 {
11867         int err;
11868
11869         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11870                 err = i40e_setup_misc_vector(pf);
11871
11872                 if (err) {
11873                         dev_info(&pf->pdev->dev,
11874                                  "MSI-X misc vector request failed, error %d\n",
11875                                  err);
11876                         return err;
11877                 }
11878         } else {
11879                 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11880
11881                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11882                                   pf->int_name, pf);
11883
11884                 if (err) {
11885                         dev_info(&pf->pdev->dev,
11886                                  "MSI/legacy misc vector request failed, error %d\n",
11887                                  err);
11888                         return err;
11889                 }
11890                 i40e_enable_misc_int_causes(pf);
11891                 i40e_irq_dynamic_enable_icr0(pf);
11892         }
11893
11894         return 0;
11895 }
11896
11897 /**
11898  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
11899  * @pf: board private structure
11900  *
11901  * This sets up the handler for MSIX 0, which is used to manage the
11902  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
11903  * when in MSI or Legacy interrupt mode.
11904  **/
11905 static int i40e_setup_misc_vector(struct i40e_pf *pf)
11906 {
11907         struct i40e_hw *hw = &pf->hw;
11908         int err = 0;
11909
11910         /* Only request the IRQ once, the first time through. */
11911         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11912                 err = request_irq(pf->msix_entries[0].vector,
11913                                   i40e_intr, 0, pf->int_name, pf);
11914                 if (err) {
11915                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11916                         dev_info(&pf->pdev->dev,
11917                                  "request_irq for %s failed: %d\n",
11918                                  pf->int_name, err);
11919                         return -EFAULT;
11920                 }
11921         }
11922
11923         i40e_enable_misc_int_causes(pf);
11924
11925         /* associate no queues to the misc vector */
11926         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11927         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
11928
11929         i40e_flush(hw);
11930
11931         i40e_irq_dynamic_enable_icr0(pf);
11932
11933         return err;
11934 }
11935
11936 /**
11937  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
11938  * @vsi: Pointer to vsi structure
11939  * @seed: Buffter to store the hash keys
11940  * @lut: Buffer to store the lookup table entries
11941  * @lut_size: Size of buffer to store the lookup table entries
11942  *
11943  * Return 0 on success, negative on failure
11944  */
11945 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11946                            u8 *lut, u16 lut_size)
11947 {
11948         struct i40e_pf *pf = vsi->back;
11949         struct i40e_hw *hw = &pf->hw;
11950         int ret = 0;
11951
11952         if (seed) {
11953                 ret = i40e_aq_get_rss_key(hw, vsi->id,
11954                         (struct i40e_aqc_get_set_rss_key_data *)seed);
11955                 if (ret) {
11956                         dev_info(&pf->pdev->dev,
11957                                  "Cannot get RSS key, err %s aq_err %s\n",
11958                                  i40e_stat_str(&pf->hw, ret),
11959                                  i40e_aq_str(&pf->hw,
11960                                              pf->hw.aq.asq_last_status));
11961                         return ret;
11962                 }
11963         }
11964
11965         if (lut) {
11966                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
11967
11968                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11969                 if (ret) {
11970                         dev_info(&pf->pdev->dev,
11971                                  "Cannot get RSS lut, err %s aq_err %s\n",
11972                                  i40e_stat_str(&pf->hw, ret),
11973                                  i40e_aq_str(&pf->hw,
11974                                              pf->hw.aq.asq_last_status));
11975                         return ret;
11976                 }
11977         }
11978
11979         return ret;
11980 }
11981
11982 /**
11983  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
11984  * @vsi: Pointer to vsi structure
11985  * @seed: RSS hash seed
11986  * @lut: Lookup table
11987  * @lut_size: Lookup table size
11988  *
11989  * Returns 0 on success, negative on failure
11990  **/
11991 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11992                                const u8 *lut, u16 lut_size)
11993 {
11994         struct i40e_pf *pf = vsi->back;
11995         struct i40e_hw *hw = &pf->hw;
11996         u16 vf_id = vsi->vf_id;
11997         u8 i;
11998
11999         /* Fill out hash function seed */
12000         if (seed) {
12001                 u32 *seed_dw = (u32 *)seed;
12002
12003                 if (vsi->type == I40E_VSI_MAIN) {
12004                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12005                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12006                 } else if (vsi->type == I40E_VSI_SRIOV) {
12007                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12008                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12009                 } else {
12010                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12011                 }
12012         }
12013
12014         if (lut) {
12015                 u32 *lut_dw = (u32 *)lut;
12016
12017                 if (vsi->type == I40E_VSI_MAIN) {
12018                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
12019                                 return -EINVAL;
12020                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12021                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12022                 } else if (vsi->type == I40E_VSI_SRIOV) {
12023                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12024                                 return -EINVAL;
12025                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12026                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12027                 } else {
12028                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12029                 }
12030         }
12031         i40e_flush(hw);
12032
12033         return 0;
12034 }
12035
12036 /**
12037  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12038  * @vsi: Pointer to VSI structure
12039  * @seed: Buffer to store the keys
12040  * @lut: Buffer to store the lookup table entries
12041  * @lut_size: Size of buffer to store the lookup table entries
12042  *
12043  * Returns 0 on success, negative on failure
12044  */
12045 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12046                             u8 *lut, u16 lut_size)
12047 {
12048         struct i40e_pf *pf = vsi->back;
12049         struct i40e_hw *hw = &pf->hw;
12050         u16 i;
12051
12052         if (seed) {
12053                 u32 *seed_dw = (u32 *)seed;
12054
12055                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12056                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12057         }
12058         if (lut) {
12059                 u32 *lut_dw = (u32 *)lut;
12060
12061                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12062                         return -EINVAL;
12063                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12064                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12065         }
12066
12067         return 0;
12068 }
12069
12070 /**
12071  * i40e_config_rss - Configure RSS keys and lut
12072  * @vsi: Pointer to VSI structure
12073  * @seed: RSS hash seed
12074  * @lut: Lookup table
12075  * @lut_size: Lookup table size
12076  *
12077  * Returns 0 on success, negative on failure
12078  */
12079 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12080 {
12081         struct i40e_pf *pf = vsi->back;
12082
12083         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12084                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12085         else
12086                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12087 }
12088
12089 /**
12090  * i40e_get_rss - Get RSS keys and lut
12091  * @vsi: Pointer to VSI structure
12092  * @seed: Buffer to store the keys
12093  * @lut: Buffer to store the lookup table entries
12094  * @lut_size: Size of buffer to store the lookup table entries
12095  *
12096  * Returns 0 on success, negative on failure
12097  */
12098 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12099 {
12100         struct i40e_pf *pf = vsi->back;
12101
12102         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12103                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12104         else
12105                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12106 }
12107
12108 /**
12109  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12110  * @pf: Pointer to board private structure
12111  * @lut: Lookup table
12112  * @rss_table_size: Lookup table size
12113  * @rss_size: Range of queue number for hashing
12114  */
12115 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12116                        u16 rss_table_size, u16 rss_size)
12117 {
12118         u16 i;
12119
12120         for (i = 0; i < rss_table_size; i++)
12121                 lut[i] = i % rss_size;
12122 }
12123
12124 /**
12125  * i40e_pf_config_rss - Prepare for RSS if used
12126  * @pf: board private structure
12127  **/
12128 static int i40e_pf_config_rss(struct i40e_pf *pf)
12129 {
12130         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12131         u8 seed[I40E_HKEY_ARRAY_SIZE];
12132         u8 *lut;
12133         struct i40e_hw *hw = &pf->hw;
12134         u32 reg_val;
12135         u64 hena;
12136         int ret;
12137
12138         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12139         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12140                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12141         hena |= i40e_pf_get_default_rss_hena(pf);
12142
12143         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12144         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12145
12146         /* Determine the RSS table size based on the hardware capabilities */
12147         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12148         reg_val = (pf->rss_table_size == 512) ?
12149                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12150                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12151         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12152
12153         /* Determine the RSS size of the VSI */
12154         if (!vsi->rss_size) {
12155                 u16 qcount;
12156                 /* If the firmware does something weird during VSI init, we
12157                  * could end up with zero TCs. Check for that to avoid
12158                  * divide-by-zero. It probably won't pass traffic, but it also
12159                  * won't panic.
12160                  */
12161                 qcount = vsi->num_queue_pairs /
12162                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12163                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12164         }
12165         if (!vsi->rss_size)
12166                 return -EINVAL;
12167
12168         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12169         if (!lut)
12170                 return -ENOMEM;
12171
12172         /* Use user configured lut if there is one, otherwise use default */
12173         if (vsi->rss_lut_user)
12174                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12175         else
12176                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12177
12178         /* Use user configured hash key if there is one, otherwise
12179          * use default.
12180          */
12181         if (vsi->rss_hkey_user)
12182                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12183         else
12184                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12185         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12186         kfree(lut);
12187
12188         return ret;
12189 }
12190
12191 /**
12192  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12193  * @pf: board private structure
12194  * @queue_count: the requested queue count for rss.
12195  *
12196  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12197  * count which may be different from the requested queue count.
12198  * Note: expects to be called while under rtnl_lock()
12199  **/
12200 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12201 {
12202         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12203         int new_rss_size;
12204
12205         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
12206                 return 0;
12207
12208         queue_count = min_t(int, queue_count, num_online_cpus());
12209         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12210
12211         if (queue_count != vsi->num_queue_pairs) {
12212                 u16 qcount;
12213
12214                 vsi->req_queue_pairs = queue_count;
12215                 i40e_prep_for_reset(pf);
12216
12217                 pf->alloc_rss_size = new_rss_size;
12218
12219                 i40e_reset_and_rebuild(pf, true, true);
12220
12221                 /* Discard the user configured hash keys and lut, if less
12222                  * queues are enabled.
12223                  */
12224                 if (queue_count < vsi->rss_size) {
12225                         i40e_clear_rss_config_user(vsi);
12226                         dev_dbg(&pf->pdev->dev,
12227                                 "discard user configured hash keys and lut\n");
12228                 }
12229
12230                 /* Reset vsi->rss_size, as number of enabled queues changed */
12231                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12232                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12233
12234                 i40e_pf_config_rss(pf);
12235         }
12236         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12237                  vsi->req_queue_pairs, pf->rss_size_max);
12238         return pf->alloc_rss_size;
12239 }
12240
12241 /**
12242  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12243  * @pf: board private structure
12244  **/
12245 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
12246 {
12247         i40e_status status;
12248         bool min_valid, max_valid;
12249         u32 max_bw, min_bw;
12250
12251         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12252                                            &min_valid, &max_valid);
12253
12254         if (!status) {
12255                 if (min_valid)
12256                         pf->min_bw = min_bw;
12257                 if (max_valid)
12258                         pf->max_bw = max_bw;
12259         }
12260
12261         return status;
12262 }
12263
12264 /**
12265  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12266  * @pf: board private structure
12267  **/
12268 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
12269 {
12270         struct i40e_aqc_configure_partition_bw_data bw_data;
12271         i40e_status status;
12272
12273         memset(&bw_data, 0, sizeof(bw_data));
12274
12275         /* Set the valid bit for this PF */
12276         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12277         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12278         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12279
12280         /* Set the new bandwidths */
12281         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12282
12283         return status;
12284 }
12285
12286 /**
12287  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12288  * @pf: board private structure
12289  **/
12290 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12291 {
12292         /* Commit temporary BW setting to permanent NVM image */
12293         enum i40e_admin_queue_err last_aq_status;
12294         i40e_status ret;
12295         u16 nvm_word;
12296
12297         if (pf->hw.partition_id != 1) {
12298                 dev_info(&pf->pdev->dev,
12299                          "Commit BW only works on partition 1! This is partition %d",
12300                          pf->hw.partition_id);
12301                 ret = I40E_NOT_SUPPORTED;
12302                 goto bw_commit_out;
12303         }
12304
12305         /* Acquire NVM for read access */
12306         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12307         last_aq_status = pf->hw.aq.asq_last_status;
12308         if (ret) {
12309                 dev_info(&pf->pdev->dev,
12310                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
12311                          i40e_stat_str(&pf->hw, ret),
12312                          i40e_aq_str(&pf->hw, last_aq_status));
12313                 goto bw_commit_out;
12314         }
12315
12316         /* Read word 0x10 of NVM - SW compatibility word 1 */
12317         ret = i40e_aq_read_nvm(&pf->hw,
12318                                I40E_SR_NVM_CONTROL_WORD,
12319                                0x10, sizeof(nvm_word), &nvm_word,
12320                                false, NULL);
12321         /* Save off last admin queue command status before releasing
12322          * the NVM
12323          */
12324         last_aq_status = pf->hw.aq.asq_last_status;
12325         i40e_release_nvm(&pf->hw);
12326         if (ret) {
12327                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
12328                          i40e_stat_str(&pf->hw, ret),
12329                          i40e_aq_str(&pf->hw, last_aq_status));
12330                 goto bw_commit_out;
12331         }
12332
12333         /* Wait a bit for NVM release to complete */
12334         msleep(50);
12335
12336         /* Acquire NVM for write access */
12337         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12338         last_aq_status = pf->hw.aq.asq_last_status;
12339         if (ret) {
12340                 dev_info(&pf->pdev->dev,
12341                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
12342                          i40e_stat_str(&pf->hw, ret),
12343                          i40e_aq_str(&pf->hw, last_aq_status));
12344                 goto bw_commit_out;
12345         }
12346         /* Write it back out unchanged to initiate update NVM,
12347          * which will force a write of the shadow (alt) RAM to
12348          * the NVM - thus storing the bandwidth values permanently.
12349          */
12350         ret = i40e_aq_update_nvm(&pf->hw,
12351                                  I40E_SR_NVM_CONTROL_WORD,
12352                                  0x10, sizeof(nvm_word),
12353                                  &nvm_word, true, 0, NULL);
12354         /* Save off last admin queue command status before releasing
12355          * the NVM
12356          */
12357         last_aq_status = pf->hw.aq.asq_last_status;
12358         i40e_release_nvm(&pf->hw);
12359         if (ret)
12360                 dev_info(&pf->pdev->dev,
12361                          "BW settings NOT SAVED, err %s aq_err %s\n",
12362                          i40e_stat_str(&pf->hw, ret),
12363                          i40e_aq_str(&pf->hw, last_aq_status));
12364 bw_commit_out:
12365
12366         return ret;
12367 }
12368
12369 /**
12370  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12371  * if total port shutdown feature is enabled for this PF
12372  * @pf: board private structure
12373  **/
12374 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12375 {
12376 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED        BIT(4)
12377 #define I40E_FEATURES_ENABLE_PTR                0x2A
12378 #define I40E_CURRENT_SETTING_PTR                0x2B
12379 #define I40E_LINK_BEHAVIOR_WORD_OFFSET          0x2D
12380 #define I40E_LINK_BEHAVIOR_WORD_LENGTH          0x1
12381 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED    BIT(0)
12382 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH      4
12383         i40e_status read_status = I40E_SUCCESS;
12384         u16 sr_emp_sr_settings_ptr = 0;
12385         u16 features_enable = 0;
12386         u16 link_behavior = 0;
12387         bool ret = false;
12388
12389         read_status = i40e_read_nvm_word(&pf->hw,
12390                                          I40E_SR_EMP_SR_SETTINGS_PTR,
12391                                          &sr_emp_sr_settings_ptr);
12392         if (read_status)
12393                 goto err_nvm;
12394         read_status = i40e_read_nvm_word(&pf->hw,
12395                                          sr_emp_sr_settings_ptr +
12396                                          I40E_FEATURES_ENABLE_PTR,
12397                                          &features_enable);
12398         if (read_status)
12399                 goto err_nvm;
12400         if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12401                 read_status = i40e_read_nvm_module_data(&pf->hw,
12402                                                         I40E_SR_EMP_SR_SETTINGS_PTR,
12403                                                         I40E_CURRENT_SETTING_PTR,
12404                                                         I40E_LINK_BEHAVIOR_WORD_OFFSET,
12405                                                         I40E_LINK_BEHAVIOR_WORD_LENGTH,
12406                                                         &link_behavior);
12407                 if (read_status)
12408                         goto err_nvm;
12409                 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12410                 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12411         }
12412         return ret;
12413
12414 err_nvm:
12415         dev_warn(&pf->pdev->dev,
12416                  "total-port-shutdown feature is off due to read nvm error: %s\n",
12417                  i40e_stat_str(&pf->hw, read_status));
12418         return ret;
12419 }
12420
12421 /**
12422  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12423  * @pf: board private structure to initialize
12424  *
12425  * i40e_sw_init initializes the Adapter private data structure.
12426  * Fields are initialized based on PCI device information and
12427  * OS network device settings (MTU size).
12428  **/
12429 static int i40e_sw_init(struct i40e_pf *pf)
12430 {
12431         int err = 0;
12432         int size;
12433         u16 pow;
12434
12435         /* Set default capability flags */
12436         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12437                     I40E_FLAG_MSI_ENABLED     |
12438                     I40E_FLAG_MSIX_ENABLED;
12439
12440         /* Set default ITR */
12441         pf->rx_itr_default = I40E_ITR_RX_DEF;
12442         pf->tx_itr_default = I40E_ITR_TX_DEF;
12443
12444         /* Depending on PF configurations, it is possible that the RSS
12445          * maximum might end up larger than the available queues
12446          */
12447         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12448         pf->alloc_rss_size = 1;
12449         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12450         pf->rss_size_max = min_t(int, pf->rss_size_max,
12451                                  pf->hw.func_caps.num_tx_qp);
12452
12453         /* find the next higher power-of-2 of num cpus */
12454         pow = roundup_pow_of_two(num_online_cpus());
12455         pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12456
12457         if (pf->hw.func_caps.rss) {
12458                 pf->flags |= I40E_FLAG_RSS_ENABLED;
12459                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12460                                            num_online_cpus());
12461         }
12462
12463         /* MFP mode enabled */
12464         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12465                 pf->flags |= I40E_FLAG_MFP_ENABLED;
12466                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12467                 if (i40e_get_partition_bw_setting(pf)) {
12468                         dev_warn(&pf->pdev->dev,
12469                                  "Could not get partition bw settings\n");
12470                 } else {
12471                         dev_info(&pf->pdev->dev,
12472                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
12473                                  pf->min_bw, pf->max_bw);
12474
12475                         /* nudge the Tx scheduler */
12476                         i40e_set_partition_bw_setting(pf);
12477                 }
12478         }
12479
12480         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12481             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12482                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12483                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12484                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12485                     pf->hw.num_partitions > 1)
12486                         dev_info(&pf->pdev->dev,
12487                                  "Flow Director Sideband mode Disabled in MFP mode\n");
12488                 else
12489                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12490                 pf->fdir_pf_filter_count =
12491                                  pf->hw.func_caps.fd_filters_guaranteed;
12492                 pf->hw.fdir_shared_filter_count =
12493                                  pf->hw.func_caps.fd_filters_best_effort;
12494         }
12495
12496         if (pf->hw.mac.type == I40E_MAC_X722) {
12497                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12498                                     I40E_HW_128_QP_RSS_CAPABLE |
12499                                     I40E_HW_ATR_EVICT_CAPABLE |
12500                                     I40E_HW_WB_ON_ITR_CAPABLE |
12501                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12502                                     I40E_HW_NO_PCI_LINK_CHECK |
12503                                     I40E_HW_USE_SET_LLDP_MIB |
12504                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12505                                     I40E_HW_PTP_L4_CAPABLE |
12506                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12507                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12508
12509 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12510                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12511                     I40E_FDEVICT_PCTYPE_DEFAULT) {
12512                         dev_warn(&pf->pdev->dev,
12513                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12514                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12515                 }
12516         } else if ((pf->hw.aq.api_maj_ver > 1) ||
12517                    ((pf->hw.aq.api_maj_ver == 1) &&
12518                     (pf->hw.aq.api_min_ver > 4))) {
12519                 /* Supported in FW API version higher than 1.4 */
12520                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12521         }
12522
12523         /* Enable HW ATR eviction if possible */
12524         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12525                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12526
12527         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12528             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12529             (pf->hw.aq.fw_maj_ver < 4))) {
12530                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12531                 /* No DCB support  for FW < v4.33 */
12532                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12533         }
12534
12535         /* Disable FW LLDP if FW < v4.3 */
12536         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12537             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12538             (pf->hw.aq.fw_maj_ver < 4)))
12539                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12540
12541         /* Use the FW Set LLDP MIB API if FW > v4.40 */
12542         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12543             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12544             (pf->hw.aq.fw_maj_ver >= 5)))
12545                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12546
12547         /* Enable PTP L4 if FW > v6.0 */
12548         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12549             pf->hw.aq.fw_maj_ver >= 6)
12550                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12551
12552         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12553                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12554                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12555                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12556         }
12557
12558         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12559                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
12560                 /* IWARP needs one extra vector for CQP just like MISC.*/
12561                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12562         }
12563         /* Stopping FW LLDP engine is supported on XL710 and X722
12564          * starting from FW versions determined in i40e_init_adminq.
12565          * Stopping the FW LLDP engine is not supported on XL710
12566          * if NPAR is functioning so unset this hw flag in this case.
12567          */
12568         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12569             pf->hw.func_caps.npar_enable &&
12570             (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12571                 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12572
12573 #ifdef CONFIG_PCI_IOV
12574         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12575                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12576                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12577                 pf->num_req_vfs = min_t(int,
12578                                         pf->hw.func_caps.num_vfs,
12579                                         I40E_MAX_VF_COUNT);
12580         }
12581 #endif /* CONFIG_PCI_IOV */
12582         pf->eeprom_version = 0xDEAD;
12583         pf->lan_veb = I40E_NO_VEB;
12584         pf->lan_vsi = I40E_NO_VSI;
12585
12586         /* By default FW has this off for performance reasons */
12587         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12588
12589         /* set up queue assignment tracking */
12590         size = sizeof(struct i40e_lump_tracking)
12591                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12592         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12593         if (!pf->qp_pile) {
12594                 err = -ENOMEM;
12595                 goto sw_init_done;
12596         }
12597         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12598         pf->qp_pile->search_hint = 0;
12599
12600         pf->tx_timeout_recovery_level = 1;
12601
12602         if (pf->hw.mac.type != I40E_MAC_X722 &&
12603             i40e_is_total_port_shutdown_enabled(pf)) {
12604                 /* Link down on close must be on when total port shutdown
12605                  * is enabled for a given port
12606                  */
12607                 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12608                               I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12609                 dev_info(&pf->pdev->dev,
12610                          "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12611         }
12612         mutex_init(&pf->switch_mutex);
12613
12614 sw_init_done:
12615         return err;
12616 }
12617
12618 /**
12619  * i40e_set_ntuple - set the ntuple feature flag and take action
12620  * @pf: board private structure to initialize
12621  * @features: the feature set that the stack is suggesting
12622  *
12623  * returns a bool to indicate if reset needs to happen
12624  **/
12625 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12626 {
12627         bool need_reset = false;
12628
12629         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12630          * the state changed, we need to reset.
12631          */
12632         if (features & NETIF_F_NTUPLE) {
12633                 /* Enable filters and mark for reset */
12634                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12635                         need_reset = true;
12636                 /* enable FD_SB only if there is MSI-X vector and no cloud
12637                  * filters exist
12638                  */
12639                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12640                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12641                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12642                 }
12643         } else {
12644                 /* turn off filters, mark for reset and clear SW filter list */
12645                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12646                         need_reset = true;
12647                         i40e_fdir_filter_exit(pf);
12648                 }
12649                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12650                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12651                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12652
12653                 /* reset fd counters */
12654                 pf->fd_add_err = 0;
12655                 pf->fd_atr_cnt = 0;
12656                 /* if ATR was auto disabled it can be re-enabled. */
12657                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12658                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12659                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12660                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12661         }
12662         return need_reset;
12663 }
12664
12665 /**
12666  * i40e_clear_rss_lut - clear the rx hash lookup table
12667  * @vsi: the VSI being configured
12668  **/
12669 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12670 {
12671         struct i40e_pf *pf = vsi->back;
12672         struct i40e_hw *hw = &pf->hw;
12673         u16 vf_id = vsi->vf_id;
12674         u8 i;
12675
12676         if (vsi->type == I40E_VSI_MAIN) {
12677                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12678                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12679         } else if (vsi->type == I40E_VSI_SRIOV) {
12680                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12681                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12682         } else {
12683                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12684         }
12685 }
12686
12687 /**
12688  * i40e_set_features - set the netdev feature flags
12689  * @netdev: ptr to the netdev being adjusted
12690  * @features: the feature set that the stack is suggesting
12691  * Note: expects to be called while under rtnl_lock()
12692  **/
12693 static int i40e_set_features(struct net_device *netdev,
12694                              netdev_features_t features)
12695 {
12696         struct i40e_netdev_priv *np = netdev_priv(netdev);
12697         struct i40e_vsi *vsi = np->vsi;
12698         struct i40e_pf *pf = vsi->back;
12699         bool need_reset;
12700
12701         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12702                 i40e_pf_config_rss(pf);
12703         else if (!(features & NETIF_F_RXHASH) &&
12704                  netdev->features & NETIF_F_RXHASH)
12705                 i40e_clear_rss_lut(vsi);
12706
12707         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12708                 i40e_vlan_stripping_enable(vsi);
12709         else
12710                 i40e_vlan_stripping_disable(vsi);
12711
12712         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12713                 dev_err(&pf->pdev->dev,
12714                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12715                 return -EINVAL;
12716         }
12717
12718         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12719                 i40e_del_all_macvlans(vsi);
12720
12721         need_reset = i40e_set_ntuple(pf, features);
12722
12723         if (need_reset)
12724                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12725
12726         return 0;
12727 }
12728
12729 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12730                                     unsigned int table, unsigned int idx,
12731                                     struct udp_tunnel_info *ti)
12732 {
12733         struct i40e_netdev_priv *np = netdev_priv(netdev);
12734         struct i40e_hw *hw = &np->vsi->back->hw;
12735         u8 type, filter_index;
12736         i40e_status ret;
12737
12738         type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12739                                                    I40E_AQC_TUNNEL_TYPE_NGE;
12740
12741         ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
12742                                      NULL);
12743         if (ret) {
12744                 netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n",
12745                             i40e_stat_str(hw, ret),
12746                             i40e_aq_str(hw, hw->aq.asq_last_status));
12747                 return -EIO;
12748         }
12749
12750         udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
12751         return 0;
12752 }
12753
12754 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12755                                       unsigned int table, unsigned int idx,
12756                                       struct udp_tunnel_info *ti)
12757 {
12758         struct i40e_netdev_priv *np = netdev_priv(netdev);
12759         struct i40e_hw *hw = &np->vsi->back->hw;
12760         i40e_status ret;
12761
12762         ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
12763         if (ret) {
12764                 netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n",
12765                             i40e_stat_str(hw, ret),
12766                             i40e_aq_str(hw, hw->aq.asq_last_status));
12767                 return -EIO;
12768         }
12769
12770         return 0;
12771 }
12772
12773 static int i40e_get_phys_port_id(struct net_device *netdev,
12774                                  struct netdev_phys_item_id *ppid)
12775 {
12776         struct i40e_netdev_priv *np = netdev_priv(netdev);
12777         struct i40e_pf *pf = np->vsi->back;
12778         struct i40e_hw *hw = &pf->hw;
12779
12780         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12781                 return -EOPNOTSUPP;
12782
12783         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12784         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12785
12786         return 0;
12787 }
12788
12789 /**
12790  * i40e_ndo_fdb_add - add an entry to the hardware database
12791  * @ndm: the input from the stack
12792  * @tb: pointer to array of nladdr (unused)
12793  * @dev: the net device pointer
12794  * @addr: the MAC address entry being added
12795  * @vid: VLAN ID
12796  * @flags: instructions from stack about fdb operation
12797  * @extack: netlink extended ack, unused currently
12798  */
12799 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12800                             struct net_device *dev,
12801                             const unsigned char *addr, u16 vid,
12802                             u16 flags,
12803                             struct netlink_ext_ack *extack)
12804 {
12805         struct i40e_netdev_priv *np = netdev_priv(dev);
12806         struct i40e_pf *pf = np->vsi->back;
12807         int err = 0;
12808
12809         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12810                 return -EOPNOTSUPP;
12811
12812         if (vid) {
12813                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12814                 return -EINVAL;
12815         }
12816
12817         /* Hardware does not support aging addresses so if a
12818          * ndm_state is given only allow permanent addresses
12819          */
12820         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12821                 netdev_info(dev, "FDB only supports static addresses\n");
12822                 return -EINVAL;
12823         }
12824
12825         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12826                 err = dev_uc_add_excl(dev, addr);
12827         else if (is_multicast_ether_addr(addr))
12828                 err = dev_mc_add_excl(dev, addr);
12829         else
12830                 err = -EINVAL;
12831
12832         /* Only return duplicate errors if NLM_F_EXCL is set */
12833         if (err == -EEXIST && !(flags & NLM_F_EXCL))
12834                 err = 0;
12835
12836         return err;
12837 }
12838
12839 /**
12840  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
12841  * @dev: the netdev being configured
12842  * @nlh: RTNL message
12843  * @flags: bridge flags
12844  * @extack: netlink extended ack
12845  *
12846  * Inserts a new hardware bridge if not already created and
12847  * enables the bridging mode requested (VEB or VEPA). If the
12848  * hardware bridge has already been inserted and the request
12849  * is to change the mode then that requires a PF reset to
12850  * allow rebuild of the components with required hardware
12851  * bridge mode enabled.
12852  *
12853  * Note: expects to be called while under rtnl_lock()
12854  **/
12855 static int i40e_ndo_bridge_setlink(struct net_device *dev,
12856                                    struct nlmsghdr *nlh,
12857                                    u16 flags,
12858                                    struct netlink_ext_ack *extack)
12859 {
12860         struct i40e_netdev_priv *np = netdev_priv(dev);
12861         struct i40e_vsi *vsi = np->vsi;
12862         struct i40e_pf *pf = vsi->back;
12863         struct i40e_veb *veb = NULL;
12864         struct nlattr *attr, *br_spec;
12865         int i, rem;
12866
12867         /* Only for PF VSI for now */
12868         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12869                 return -EOPNOTSUPP;
12870
12871         /* Find the HW bridge for PF VSI */
12872         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12873                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12874                         veb = pf->veb[i];
12875         }
12876
12877         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12878
12879         nla_for_each_nested(attr, br_spec, rem) {
12880                 __u16 mode;
12881
12882                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12883                         continue;
12884
12885                 mode = nla_get_u16(attr);
12886                 if ((mode != BRIDGE_MODE_VEPA) &&
12887                     (mode != BRIDGE_MODE_VEB))
12888                         return -EINVAL;
12889
12890                 /* Insert a new HW bridge */
12891                 if (!veb) {
12892                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12893                                              vsi->tc_config.enabled_tc);
12894                         if (veb) {
12895                                 veb->bridge_mode = mode;
12896                                 i40e_config_bridge_mode(veb);
12897                         } else {
12898                                 /* No Bridge HW offload available */
12899                                 return -ENOENT;
12900                         }
12901                         break;
12902                 } else if (mode != veb->bridge_mode) {
12903                         /* Existing HW bridge but different mode needs reset */
12904                         veb->bridge_mode = mode;
12905                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
12906                         if (mode == BRIDGE_MODE_VEB)
12907                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12908                         else
12909                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12910                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12911                         break;
12912                 }
12913         }
12914
12915         return 0;
12916 }
12917
12918 /**
12919  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
12920  * @skb: skb buff
12921  * @pid: process id
12922  * @seq: RTNL message seq #
12923  * @dev: the netdev being configured
12924  * @filter_mask: unused
12925  * @nlflags: netlink flags passed in
12926  *
12927  * Return the mode in which the hardware bridge is operating in
12928  * i.e VEB or VEPA.
12929  **/
12930 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12931                                    struct net_device *dev,
12932                                    u32 __always_unused filter_mask,
12933                                    int nlflags)
12934 {
12935         struct i40e_netdev_priv *np = netdev_priv(dev);
12936         struct i40e_vsi *vsi = np->vsi;
12937         struct i40e_pf *pf = vsi->back;
12938         struct i40e_veb *veb = NULL;
12939         int i;
12940
12941         /* Only for PF VSI for now */
12942         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12943                 return -EOPNOTSUPP;
12944
12945         /* Find the HW bridge for the PF VSI */
12946         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12947                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12948                         veb = pf->veb[i];
12949         }
12950
12951         if (!veb)
12952                 return 0;
12953
12954         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12955                                        0, 0, nlflags, filter_mask, NULL);
12956 }
12957
12958 /**
12959  * i40e_features_check - Validate encapsulated packet conforms to limits
12960  * @skb: skb buff
12961  * @dev: This physical port's netdev
12962  * @features: Offload features that the stack believes apply
12963  **/
12964 static netdev_features_t i40e_features_check(struct sk_buff *skb,
12965                                              struct net_device *dev,
12966                                              netdev_features_t features)
12967 {
12968         size_t len;
12969
12970         /* No point in doing any of this if neither checksum nor GSO are
12971          * being requested for this frame.  We can rule out both by just
12972          * checking for CHECKSUM_PARTIAL
12973          */
12974         if (skb->ip_summed != CHECKSUM_PARTIAL)
12975                 return features;
12976
12977         /* We cannot support GSO if the MSS is going to be less than
12978          * 64 bytes.  If it is then we need to drop support for GSO.
12979          */
12980         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12981                 features &= ~NETIF_F_GSO_MASK;
12982
12983         /* MACLEN can support at most 63 words */
12984         len = skb_network_header(skb) - skb->data;
12985         if (len & ~(63 * 2))
12986                 goto out_err;
12987
12988         /* IPLEN and EIPLEN can support at most 127 dwords */
12989         len = skb_transport_header(skb) - skb_network_header(skb);
12990         if (len & ~(127 * 4))
12991                 goto out_err;
12992
12993         if (skb->encapsulation) {
12994                 /* L4TUNLEN can support 127 words */
12995                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12996                 if (len & ~(127 * 2))
12997                         goto out_err;
12998
12999                 /* IPLEN can support at most 127 dwords */
13000                 len = skb_inner_transport_header(skb) -
13001                       skb_inner_network_header(skb);
13002                 if (len & ~(127 * 4))
13003                         goto out_err;
13004         }
13005
13006         /* No need to validate L4LEN as TCP is the only protocol with a
13007          * a flexible value and we support all possible values supported
13008          * by TCP, which is at most 15 dwords
13009          */
13010
13011         return features;
13012 out_err:
13013         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13014 }
13015
13016 /**
13017  * i40e_xdp_setup - add/remove an XDP program
13018  * @vsi: VSI to changed
13019  * @prog: XDP program
13020  * @extack: netlink extended ack
13021  **/
13022 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13023                           struct netlink_ext_ack *extack)
13024 {
13025         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
13026         struct i40e_pf *pf = vsi->back;
13027         struct bpf_prog *old_prog;
13028         bool need_reset;
13029         int i;
13030
13031         /* Don't allow frames that span over multiple buffers */
13032         if (frame_size > vsi->rx_buf_len) {
13033                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
13034                 return -EINVAL;
13035         }
13036
13037         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
13038         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13039
13040         if (need_reset)
13041                 i40e_prep_for_reset(pf);
13042
13043         old_prog = xchg(&vsi->xdp_prog, prog);
13044
13045         if (need_reset) {
13046                 if (!prog)
13047                         /* Wait until ndo_xsk_wakeup completes. */
13048                         synchronize_rcu();
13049                 i40e_reset_and_rebuild(pf, true, true);
13050         }
13051
13052         for (i = 0; i < vsi->num_queue_pairs; i++)
13053                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13054
13055         if (old_prog)
13056                 bpf_prog_put(old_prog);
13057
13058         /* Kick start the NAPI context if there is an AF_XDP socket open
13059          * on that queue id. This so that receiving will start.
13060          */
13061         if (need_reset && prog)
13062                 for (i = 0; i < vsi->num_queue_pairs; i++)
13063                         if (vsi->xdp_rings[i]->xsk_pool)
13064                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
13065                                                       XDP_WAKEUP_RX);
13066
13067         return 0;
13068 }
13069
13070 /**
13071  * i40e_enter_busy_conf - Enters busy config state
13072  * @vsi: vsi
13073  *
13074  * Returns 0 on success, <0 for failure.
13075  **/
13076 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13077 {
13078         struct i40e_pf *pf = vsi->back;
13079         int timeout = 50;
13080
13081         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13082                 timeout--;
13083                 if (!timeout)
13084                         return -EBUSY;
13085                 usleep_range(1000, 2000);
13086         }
13087
13088         return 0;
13089 }
13090
13091 /**
13092  * i40e_exit_busy_conf - Exits busy config state
13093  * @vsi: vsi
13094  **/
13095 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13096 {
13097         struct i40e_pf *pf = vsi->back;
13098
13099         clear_bit(__I40E_CONFIG_BUSY, pf->state);
13100 }
13101
13102 /**
13103  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13104  * @vsi: vsi
13105  * @queue_pair: queue pair
13106  **/
13107 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13108 {
13109         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13110                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13111         memset(&vsi->tx_rings[queue_pair]->stats, 0,
13112                sizeof(vsi->tx_rings[queue_pair]->stats));
13113         if (i40e_enabled_xdp_vsi(vsi)) {
13114                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13115                        sizeof(vsi->xdp_rings[queue_pair]->stats));
13116         }
13117 }
13118
13119 /**
13120  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13121  * @vsi: vsi
13122  * @queue_pair: queue pair
13123  **/
13124 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13125 {
13126         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13127         if (i40e_enabled_xdp_vsi(vsi)) {
13128                 /* Make sure that in-progress ndo_xdp_xmit calls are
13129                  * completed.
13130                  */
13131                 synchronize_rcu();
13132                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13133         }
13134         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13135 }
13136
13137 /**
13138  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13139  * @vsi: vsi
13140  * @queue_pair: queue pair
13141  * @enable: true for enable, false for disable
13142  **/
13143 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13144                                         bool enable)
13145 {
13146         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13147         struct i40e_q_vector *q_vector = rxr->q_vector;
13148
13149         if (!vsi->netdev)
13150                 return;
13151
13152         /* All rings in a qp belong to the same qvector. */
13153         if (q_vector->rx.ring || q_vector->tx.ring) {
13154                 if (enable)
13155                         napi_enable(&q_vector->napi);
13156                 else
13157                         napi_disable(&q_vector->napi);
13158         }
13159 }
13160
13161 /**
13162  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13163  * @vsi: vsi
13164  * @queue_pair: queue pair
13165  * @enable: true for enable, false for disable
13166  *
13167  * Returns 0 on success, <0 on failure.
13168  **/
13169 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13170                                         bool enable)
13171 {
13172         struct i40e_pf *pf = vsi->back;
13173         int pf_q, ret = 0;
13174
13175         pf_q = vsi->base_queue + queue_pair;
13176         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13177                                      false /*is xdp*/, enable);
13178         if (ret) {
13179                 dev_info(&pf->pdev->dev,
13180                          "VSI seid %d Tx ring %d %sable timeout\n",
13181                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13182                 return ret;
13183         }
13184
13185         i40e_control_rx_q(pf, pf_q, enable);
13186         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13187         if (ret) {
13188                 dev_info(&pf->pdev->dev,
13189                          "VSI seid %d Rx ring %d %sable timeout\n",
13190                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13191                 return ret;
13192         }
13193
13194         /* Due to HW errata, on Rx disable only, the register can
13195          * indicate done before it really is. Needs 50ms to be sure
13196          */
13197         if (!enable)
13198                 mdelay(50);
13199
13200         if (!i40e_enabled_xdp_vsi(vsi))
13201                 return ret;
13202
13203         ret = i40e_control_wait_tx_q(vsi->seid, pf,
13204                                      pf_q + vsi->alloc_queue_pairs,
13205                                      true /*is xdp*/, enable);
13206         if (ret) {
13207                 dev_info(&pf->pdev->dev,
13208                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
13209                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13210         }
13211
13212         return ret;
13213 }
13214
13215 /**
13216  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13217  * @vsi: vsi
13218  * @queue_pair: queue_pair
13219  **/
13220 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13221 {
13222         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13223         struct i40e_pf *pf = vsi->back;
13224         struct i40e_hw *hw = &pf->hw;
13225
13226         /* All rings in a qp belong to the same qvector. */
13227         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
13228                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13229         else
13230                 i40e_irq_dynamic_enable_icr0(pf);
13231
13232         i40e_flush(hw);
13233 }
13234
13235 /**
13236  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13237  * @vsi: vsi
13238  * @queue_pair: queue_pair
13239  **/
13240 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13241 {
13242         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13243         struct i40e_pf *pf = vsi->back;
13244         struct i40e_hw *hw = &pf->hw;
13245
13246         /* For simplicity, instead of removing the qp interrupt causes
13247          * from the interrupt linked list, we simply disable the interrupt, and
13248          * leave the list intact.
13249          *
13250          * All rings in a qp belong to the same qvector.
13251          */
13252         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13253                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13254
13255                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13256                 i40e_flush(hw);
13257                 synchronize_irq(pf->msix_entries[intpf].vector);
13258         } else {
13259                 /* Legacy and MSI mode - this stops all interrupt handling */
13260                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13261                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13262                 i40e_flush(hw);
13263                 synchronize_irq(pf->pdev->irq);
13264         }
13265 }
13266
13267 /**
13268  * i40e_queue_pair_disable - Disables a queue pair
13269  * @vsi: vsi
13270  * @queue_pair: queue pair
13271  *
13272  * Returns 0 on success, <0 on failure.
13273  **/
13274 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13275 {
13276         int err;
13277
13278         err = i40e_enter_busy_conf(vsi);
13279         if (err)
13280                 return err;
13281
13282         i40e_queue_pair_disable_irq(vsi, queue_pair);
13283         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13284         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13285         i40e_queue_pair_clean_rings(vsi, queue_pair);
13286         i40e_queue_pair_reset_stats(vsi, queue_pair);
13287
13288         return err;
13289 }
13290
13291 /**
13292  * i40e_queue_pair_enable - Enables a queue pair
13293  * @vsi: vsi
13294  * @queue_pair: queue pair
13295  *
13296  * Returns 0 on success, <0 on failure.
13297  **/
13298 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13299 {
13300         int err;
13301
13302         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13303         if (err)
13304                 return err;
13305
13306         if (i40e_enabled_xdp_vsi(vsi)) {
13307                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13308                 if (err)
13309                         return err;
13310         }
13311
13312         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13313         if (err)
13314                 return err;
13315
13316         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13317         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13318         i40e_queue_pair_enable_irq(vsi, queue_pair);
13319
13320         i40e_exit_busy_conf(vsi);
13321
13322         return err;
13323 }
13324
13325 /**
13326  * i40e_xdp - implements ndo_bpf for i40e
13327  * @dev: netdevice
13328  * @xdp: XDP command
13329  **/
13330 static int i40e_xdp(struct net_device *dev,
13331                     struct netdev_bpf *xdp)
13332 {
13333         struct i40e_netdev_priv *np = netdev_priv(dev);
13334         struct i40e_vsi *vsi = np->vsi;
13335
13336         if (vsi->type != I40E_VSI_MAIN)
13337                 return -EINVAL;
13338
13339         switch (xdp->command) {
13340         case XDP_SETUP_PROG:
13341                 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13342         case XDP_SETUP_XSK_POOL:
13343                 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13344                                            xdp->xsk.queue_id);
13345         default:
13346                 return -EINVAL;
13347         }
13348 }
13349
13350 static const struct net_device_ops i40e_netdev_ops = {
13351         .ndo_open               = i40e_open,
13352         .ndo_stop               = i40e_close,
13353         .ndo_start_xmit         = i40e_lan_xmit_frame,
13354         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
13355         .ndo_set_rx_mode        = i40e_set_rx_mode,
13356         .ndo_validate_addr      = eth_validate_addr,
13357         .ndo_set_mac_address    = i40e_set_mac,
13358         .ndo_change_mtu         = i40e_change_mtu,
13359         .ndo_eth_ioctl          = i40e_ioctl,
13360         .ndo_tx_timeout         = i40e_tx_timeout,
13361         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
13362         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
13363 #ifdef CONFIG_NET_POLL_CONTROLLER
13364         .ndo_poll_controller    = i40e_netpoll,
13365 #endif
13366         .ndo_setup_tc           = __i40e_setup_tc,
13367         .ndo_select_queue       = i40e_lan_select_queue,
13368         .ndo_set_features       = i40e_set_features,
13369         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
13370         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
13371         .ndo_get_vf_stats       = i40e_get_vf_stats,
13372         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
13373         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
13374         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
13375         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
13376         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
13377         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
13378         .ndo_fdb_add            = i40e_ndo_fdb_add,
13379         .ndo_features_check     = i40e_features_check,
13380         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
13381         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
13382         .ndo_bpf                = i40e_xdp,
13383         .ndo_xdp_xmit           = i40e_xdp_xmit,
13384         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
13385         .ndo_dfwd_add_station   = i40e_fwd_add,
13386         .ndo_dfwd_del_station   = i40e_fwd_del,
13387 };
13388
13389 /**
13390  * i40e_config_netdev - Setup the netdev flags
13391  * @vsi: the VSI being configured
13392  *
13393  * Returns 0 on success, negative value on failure
13394  **/
13395 static int i40e_config_netdev(struct i40e_vsi *vsi)
13396 {
13397         struct i40e_pf *pf = vsi->back;
13398         struct i40e_hw *hw = &pf->hw;
13399         struct i40e_netdev_priv *np;
13400         struct net_device *netdev;
13401         u8 broadcast[ETH_ALEN];
13402         u8 mac_addr[ETH_ALEN];
13403         int etherdev_size;
13404         netdev_features_t hw_enc_features;
13405         netdev_features_t hw_features;
13406
13407         etherdev_size = sizeof(struct i40e_netdev_priv);
13408         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13409         if (!netdev)
13410                 return -ENOMEM;
13411
13412         vsi->netdev = netdev;
13413         np = netdev_priv(netdev);
13414         np->vsi = vsi;
13415
13416         hw_enc_features = NETIF_F_SG                    |
13417                           NETIF_F_IP_CSUM               |
13418                           NETIF_F_IPV6_CSUM             |
13419                           NETIF_F_HIGHDMA               |
13420                           NETIF_F_SOFT_FEATURES         |
13421                           NETIF_F_TSO                   |
13422                           NETIF_F_TSO_ECN               |
13423                           NETIF_F_TSO6                  |
13424                           NETIF_F_GSO_GRE               |
13425                           NETIF_F_GSO_GRE_CSUM          |
13426                           NETIF_F_GSO_PARTIAL           |
13427                           NETIF_F_GSO_IPXIP4            |
13428                           NETIF_F_GSO_IPXIP6            |
13429                           NETIF_F_GSO_UDP_TUNNEL        |
13430                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
13431                           NETIF_F_GSO_UDP_L4            |
13432                           NETIF_F_SCTP_CRC              |
13433                           NETIF_F_RXHASH                |
13434                           NETIF_F_RXCSUM                |
13435                           0;
13436
13437         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13438                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13439
13440         netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13441
13442         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13443
13444         netdev->hw_enc_features |= hw_enc_features;
13445
13446         /* record features VLANs can make use of */
13447         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13448
13449         /* enable macvlan offloads */
13450         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13451
13452         hw_features = hw_enc_features           |
13453                       NETIF_F_HW_VLAN_CTAG_TX   |
13454                       NETIF_F_HW_VLAN_CTAG_RX;
13455
13456         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13457                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13458
13459         netdev->hw_features |= hw_features;
13460
13461         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13462         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13463
13464         if (vsi->type == I40E_VSI_MAIN) {
13465                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13466                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13467                 /* The following steps are necessary for two reasons. First,
13468                  * some older NVM configurations load a default MAC-VLAN
13469                  * filter that will accept any tagged packet, and we want to
13470                  * replace this with a normal filter. Additionally, it is
13471                  * possible our MAC address was provided by the platform using
13472                  * Open Firmware or similar.
13473                  *
13474                  * Thus, we need to remove the default filter and install one
13475                  * specific to the MAC address.
13476                  */
13477                 i40e_rm_default_mac_filter(vsi, mac_addr);
13478                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13479                 i40e_add_mac_filter(vsi, mac_addr);
13480                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13481         } else {
13482                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13483                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13484                  * the end, which is 4 bytes long, so force truncation of the
13485                  * original name by IFNAMSIZ - 4
13486                  */
13487                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13488                          IFNAMSIZ - 4,
13489                          pf->vsi[pf->lan_vsi]->netdev->name);
13490                 eth_random_addr(mac_addr);
13491
13492                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13493                 i40e_add_mac_filter(vsi, mac_addr);
13494                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13495         }
13496
13497         /* Add the broadcast filter so that we initially will receive
13498          * broadcast packets. Note that when a new VLAN is first added the
13499          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13500          * specific filters as part of transitioning into "vlan" operation.
13501          * When more VLANs are added, the driver will copy each existing MAC
13502          * filter and add it for the new VLAN.
13503          *
13504          * Broadcast filters are handled specially by
13505          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13506          * promiscuous bit instead of adding this directly as a MAC/VLAN
13507          * filter. The subtask will update the correct broadcast promiscuous
13508          * bits as VLANs become active or inactive.
13509          */
13510         eth_broadcast_addr(broadcast);
13511         spin_lock_bh(&vsi->mac_filter_hash_lock);
13512         i40e_add_mac_filter(vsi, broadcast);
13513         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13514
13515         eth_hw_addr_set(netdev, mac_addr);
13516         ether_addr_copy(netdev->perm_addr, mac_addr);
13517
13518         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13519         netdev->neigh_priv_len = sizeof(u32) * 4;
13520
13521         netdev->priv_flags |= IFF_UNICAST_FLT;
13522         netdev->priv_flags |= IFF_SUPP_NOFCS;
13523         /* Setup netdev TC information */
13524         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13525
13526         netdev->netdev_ops = &i40e_netdev_ops;
13527         netdev->watchdog_timeo = 5 * HZ;
13528         i40e_set_ethtool_ops(netdev);
13529
13530         /* MTU range: 68 - 9706 */
13531         netdev->min_mtu = ETH_MIN_MTU;
13532         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13533
13534         return 0;
13535 }
13536
13537 /**
13538  * i40e_vsi_delete - Delete a VSI from the switch
13539  * @vsi: the VSI being removed
13540  *
13541  * Returns 0 on success, negative value on failure
13542  **/
13543 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13544 {
13545         /* remove default VSI is not allowed */
13546         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13547                 return;
13548
13549         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13550 }
13551
13552 /**
13553  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13554  * @vsi: the VSI being queried
13555  *
13556  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13557  **/
13558 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13559 {
13560         struct i40e_veb *veb;
13561         struct i40e_pf *pf = vsi->back;
13562
13563         /* Uplink is not a bridge so default to VEB */
13564         if (vsi->veb_idx >= I40E_MAX_VEB)
13565                 return 1;
13566
13567         veb = pf->veb[vsi->veb_idx];
13568         if (!veb) {
13569                 dev_info(&pf->pdev->dev,
13570                          "There is no veb associated with the bridge\n");
13571                 return -ENOENT;
13572         }
13573
13574         /* Uplink is a bridge in VEPA mode */
13575         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13576                 return 0;
13577         } else {
13578                 /* Uplink is a bridge in VEB mode */
13579                 return 1;
13580         }
13581
13582         /* VEPA is now default bridge, so return 0 */
13583         return 0;
13584 }
13585
13586 /**
13587  * i40e_add_vsi - Add a VSI to the switch
13588  * @vsi: the VSI being configured
13589  *
13590  * This initializes a VSI context depending on the VSI type to be added and
13591  * passes it down to the add_vsi aq command.
13592  **/
13593 static int i40e_add_vsi(struct i40e_vsi *vsi)
13594 {
13595         int ret = -ENODEV;
13596         struct i40e_pf *pf = vsi->back;
13597         struct i40e_hw *hw = &pf->hw;
13598         struct i40e_vsi_context ctxt;
13599         struct i40e_mac_filter *f;
13600         struct hlist_node *h;
13601         int bkt;
13602
13603         u8 enabled_tc = 0x1; /* TC0 enabled */
13604         int f_count = 0;
13605
13606         memset(&ctxt, 0, sizeof(ctxt));
13607         switch (vsi->type) {
13608         case I40E_VSI_MAIN:
13609                 /* The PF's main VSI is already setup as part of the
13610                  * device initialization, so we'll not bother with
13611                  * the add_vsi call, but we will retrieve the current
13612                  * VSI context.
13613                  */
13614                 ctxt.seid = pf->main_vsi_seid;
13615                 ctxt.pf_num = pf->hw.pf_id;
13616                 ctxt.vf_num = 0;
13617                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13618                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13619                 if (ret) {
13620                         dev_info(&pf->pdev->dev,
13621                                  "couldn't get PF vsi config, err %s aq_err %s\n",
13622                                  i40e_stat_str(&pf->hw, ret),
13623                                  i40e_aq_str(&pf->hw,
13624                                              pf->hw.aq.asq_last_status));
13625                         return -ENOENT;
13626                 }
13627                 vsi->info = ctxt.info;
13628                 vsi->info.valid_sections = 0;
13629
13630                 vsi->seid = ctxt.seid;
13631                 vsi->id = ctxt.vsi_number;
13632
13633                 enabled_tc = i40e_pf_get_tc_map(pf);
13634
13635                 /* Source pruning is enabled by default, so the flag is
13636                  * negative logic - if it's set, we need to fiddle with
13637                  * the VSI to disable source pruning.
13638                  */
13639                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13640                         memset(&ctxt, 0, sizeof(ctxt));
13641                         ctxt.seid = pf->main_vsi_seid;
13642                         ctxt.pf_num = pf->hw.pf_id;
13643                         ctxt.vf_num = 0;
13644                         ctxt.info.valid_sections |=
13645                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13646                         ctxt.info.switch_id =
13647                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13648                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13649                         if (ret) {
13650                                 dev_info(&pf->pdev->dev,
13651                                          "update vsi failed, err %s aq_err %s\n",
13652                                          i40e_stat_str(&pf->hw, ret),
13653                                          i40e_aq_str(&pf->hw,
13654                                                      pf->hw.aq.asq_last_status));
13655                                 ret = -ENOENT;
13656                                 goto err;
13657                         }
13658                 }
13659
13660                 /* MFP mode setup queue map and update VSI */
13661                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13662                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13663                         memset(&ctxt, 0, sizeof(ctxt));
13664                         ctxt.seid = pf->main_vsi_seid;
13665                         ctxt.pf_num = pf->hw.pf_id;
13666                         ctxt.vf_num = 0;
13667                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13668                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13669                         if (ret) {
13670                                 dev_info(&pf->pdev->dev,
13671                                          "update vsi failed, err %s aq_err %s\n",
13672                                          i40e_stat_str(&pf->hw, ret),
13673                                          i40e_aq_str(&pf->hw,
13674                                                     pf->hw.aq.asq_last_status));
13675                                 ret = -ENOENT;
13676                                 goto err;
13677                         }
13678                         /* update the local VSI info queue map */
13679                         i40e_vsi_update_queue_map(vsi, &ctxt);
13680                         vsi->info.valid_sections = 0;
13681                 } else {
13682                         /* Default/Main VSI is only enabled for TC0
13683                          * reconfigure it to enable all TCs that are
13684                          * available on the port in SFP mode.
13685                          * For MFP case the iSCSI PF would use this
13686                          * flow to enable LAN+iSCSI TC.
13687                          */
13688                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13689                         if (ret) {
13690                                 /* Single TC condition is not fatal,
13691                                  * message and continue
13692                                  */
13693                                 dev_info(&pf->pdev->dev,
13694                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13695                                          enabled_tc,
13696                                          i40e_stat_str(&pf->hw, ret),
13697                                          i40e_aq_str(&pf->hw,
13698                                                     pf->hw.aq.asq_last_status));
13699                         }
13700                 }
13701                 break;
13702
13703         case I40E_VSI_FDIR:
13704                 ctxt.pf_num = hw->pf_id;
13705                 ctxt.vf_num = 0;
13706                 ctxt.uplink_seid = vsi->uplink_seid;
13707                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13708                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13709                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13710                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13711                         ctxt.info.valid_sections |=
13712                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13713                         ctxt.info.switch_id =
13714                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13715                 }
13716                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13717                 break;
13718
13719         case I40E_VSI_VMDQ2:
13720                 ctxt.pf_num = hw->pf_id;
13721                 ctxt.vf_num = 0;
13722                 ctxt.uplink_seid = vsi->uplink_seid;
13723                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13724                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13725
13726                 /* This VSI is connected to VEB so the switch_id
13727                  * should be set to zero by default.
13728                  */
13729                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13730                         ctxt.info.valid_sections |=
13731                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13732                         ctxt.info.switch_id =
13733                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13734                 }
13735
13736                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13737                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13738                 break;
13739
13740         case I40E_VSI_SRIOV:
13741                 ctxt.pf_num = hw->pf_id;
13742                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13743                 ctxt.uplink_seid = vsi->uplink_seid;
13744                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13745                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13746
13747                 /* This VSI is connected to VEB so the switch_id
13748                  * should be set to zero by default.
13749                  */
13750                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13751                         ctxt.info.valid_sections |=
13752                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13753                         ctxt.info.switch_id =
13754                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13755                 }
13756
13757                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13758                         ctxt.info.valid_sections |=
13759                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13760                         ctxt.info.queueing_opt_flags |=
13761                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13762                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13763                 }
13764
13765                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13766                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13767                 if (pf->vf[vsi->vf_id].spoofchk) {
13768                         ctxt.info.valid_sections |=
13769                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13770                         ctxt.info.sec_flags |=
13771                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13772                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13773                 }
13774                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13775                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13776                 break;
13777
13778         case I40E_VSI_IWARP:
13779                 /* send down message to iWARP */
13780                 break;
13781
13782         default:
13783                 return -ENODEV;
13784         }
13785
13786         if (vsi->type != I40E_VSI_MAIN) {
13787                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13788                 if (ret) {
13789                         dev_info(&vsi->back->pdev->dev,
13790                                  "add vsi failed, err %s aq_err %s\n",
13791                                  i40e_stat_str(&pf->hw, ret),
13792                                  i40e_aq_str(&pf->hw,
13793                                              pf->hw.aq.asq_last_status));
13794                         ret = -ENOENT;
13795                         goto err;
13796                 }
13797                 vsi->info = ctxt.info;
13798                 vsi->info.valid_sections = 0;
13799                 vsi->seid = ctxt.seid;
13800                 vsi->id = ctxt.vsi_number;
13801         }
13802
13803         vsi->active_filters = 0;
13804         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13805         spin_lock_bh(&vsi->mac_filter_hash_lock);
13806         /* If macvlan filters already exist, force them to get loaded */
13807         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13808                 f->state = I40E_FILTER_NEW;
13809                 f_count++;
13810         }
13811         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13812
13813         if (f_count) {
13814                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13815                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13816         }
13817
13818         /* Update VSI BW information */
13819         ret = i40e_vsi_get_bw_info(vsi);
13820         if (ret) {
13821                 dev_info(&pf->pdev->dev,
13822                          "couldn't get vsi bw info, err %s aq_err %s\n",
13823                          i40e_stat_str(&pf->hw, ret),
13824                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13825                 /* VSI is already added so not tearing that up */
13826                 ret = 0;
13827         }
13828
13829 err:
13830         return ret;
13831 }
13832
13833 /**
13834  * i40e_vsi_release - Delete a VSI and free its resources
13835  * @vsi: the VSI being removed
13836  *
13837  * Returns 0 on success or < 0 on error
13838  **/
13839 int i40e_vsi_release(struct i40e_vsi *vsi)
13840 {
13841         struct i40e_mac_filter *f;
13842         struct hlist_node *h;
13843         struct i40e_veb *veb = NULL;
13844         struct i40e_pf *pf;
13845         u16 uplink_seid;
13846         int i, n, bkt;
13847
13848         pf = vsi->back;
13849
13850         /* release of a VEB-owner or last VSI is not allowed */
13851         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13852                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13853                          vsi->seid, vsi->uplink_seid);
13854                 return -ENODEV;
13855         }
13856         if (vsi == pf->vsi[pf->lan_vsi] &&
13857             !test_bit(__I40E_DOWN, pf->state)) {
13858                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13859                 return -ENODEV;
13860         }
13861         set_bit(__I40E_VSI_RELEASING, vsi->state);
13862         uplink_seid = vsi->uplink_seid;
13863         if (vsi->type != I40E_VSI_SRIOV) {
13864                 if (vsi->netdev_registered) {
13865                         vsi->netdev_registered = false;
13866                         if (vsi->netdev) {
13867                                 /* results in a call to i40e_close() */
13868                                 unregister_netdev(vsi->netdev);
13869                         }
13870                 } else {
13871                         i40e_vsi_close(vsi);
13872                 }
13873                 i40e_vsi_disable_irq(vsi);
13874         }
13875
13876         spin_lock_bh(&vsi->mac_filter_hash_lock);
13877
13878         /* clear the sync flag on all filters */
13879         if (vsi->netdev) {
13880                 __dev_uc_unsync(vsi->netdev, NULL);
13881                 __dev_mc_unsync(vsi->netdev, NULL);
13882         }
13883
13884         /* make sure any remaining filters are marked for deletion */
13885         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13886                 __i40e_del_filter(vsi, f);
13887
13888         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13889
13890         i40e_sync_vsi_filters(vsi);
13891
13892         i40e_vsi_delete(vsi);
13893         i40e_vsi_free_q_vectors(vsi);
13894         if (vsi->netdev) {
13895                 free_netdev(vsi->netdev);
13896                 vsi->netdev = NULL;
13897         }
13898         i40e_vsi_clear_rings(vsi);
13899         i40e_vsi_clear(vsi);
13900
13901         /* If this was the last thing on the VEB, except for the
13902          * controlling VSI, remove the VEB, which puts the controlling
13903          * VSI onto the next level down in the switch.
13904          *
13905          * Well, okay, there's one more exception here: don't remove
13906          * the orphan VEBs yet.  We'll wait for an explicit remove request
13907          * from up the network stack.
13908          */
13909         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13910                 if (pf->vsi[i] &&
13911                     pf->vsi[i]->uplink_seid == uplink_seid &&
13912                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13913                         n++;      /* count the VSIs */
13914                 }
13915         }
13916         for (i = 0; i < I40E_MAX_VEB; i++) {
13917                 if (!pf->veb[i])
13918                         continue;
13919                 if (pf->veb[i]->uplink_seid == uplink_seid)
13920                         n++;     /* count the VEBs */
13921                 if (pf->veb[i]->seid == uplink_seid)
13922                         veb = pf->veb[i];
13923         }
13924         if (n == 0 && veb && veb->uplink_seid != 0)
13925                 i40e_veb_release(veb);
13926
13927         return 0;
13928 }
13929
13930 /**
13931  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
13932  * @vsi: ptr to the VSI
13933  *
13934  * This should only be called after i40e_vsi_mem_alloc() which allocates the
13935  * corresponding SW VSI structure and initializes num_queue_pairs for the
13936  * newly allocated VSI.
13937  *
13938  * Returns 0 on success or negative on failure
13939  **/
13940 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13941 {
13942         int ret = -ENOENT;
13943         struct i40e_pf *pf = vsi->back;
13944
13945         if (vsi->q_vectors[0]) {
13946                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13947                          vsi->seid);
13948                 return -EEXIST;
13949         }
13950
13951         if (vsi->base_vector) {
13952                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13953                          vsi->seid, vsi->base_vector);
13954                 return -EEXIST;
13955         }
13956
13957         ret = i40e_vsi_alloc_q_vectors(vsi);
13958         if (ret) {
13959                 dev_info(&pf->pdev->dev,
13960                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13961                          vsi->num_q_vectors, vsi->seid, ret);
13962                 vsi->num_q_vectors = 0;
13963                 goto vector_setup_out;
13964         }
13965
13966         /* In Legacy mode, we do not have to get any other vector since we
13967          * piggyback on the misc/ICR0 for queue interrupts.
13968         */
13969         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13970                 return ret;
13971         if (vsi->num_q_vectors)
13972                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13973                                                  vsi->num_q_vectors, vsi->idx);
13974         if (vsi->base_vector < 0) {
13975                 dev_info(&pf->pdev->dev,
13976                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13977                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13978                 i40e_vsi_free_q_vectors(vsi);
13979                 ret = -ENOENT;
13980                 goto vector_setup_out;
13981         }
13982
13983 vector_setup_out:
13984         return ret;
13985 }
13986
13987 /**
13988  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
13989  * @vsi: pointer to the vsi.
13990  *
13991  * This re-allocates a vsi's queue resources.
13992  *
13993  * Returns pointer to the successfully allocated and configured VSI sw struct
13994  * on success, otherwise returns NULL on failure.
13995  **/
13996 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13997 {
13998         u16 alloc_queue_pairs;
13999         struct i40e_pf *pf;
14000         u8 enabled_tc;
14001         int ret;
14002
14003         if (!vsi)
14004                 return NULL;
14005
14006         pf = vsi->back;
14007
14008         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14009         i40e_vsi_clear_rings(vsi);
14010
14011         i40e_vsi_free_arrays(vsi, false);
14012         i40e_set_num_rings_in_vsi(vsi);
14013         ret = i40e_vsi_alloc_arrays(vsi, false);
14014         if (ret)
14015                 goto err_vsi;
14016
14017         alloc_queue_pairs = vsi->alloc_queue_pairs *
14018                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14019
14020         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14021         if (ret < 0) {
14022                 dev_info(&pf->pdev->dev,
14023                          "failed to get tracking for %d queues for VSI %d err %d\n",
14024                          alloc_queue_pairs, vsi->seid, ret);
14025                 goto err_vsi;
14026         }
14027         vsi->base_queue = ret;
14028
14029         /* Update the FW view of the VSI. Force a reset of TC and queue
14030          * layout configurations.
14031          */
14032         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14033         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14034         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14035         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14036         if (vsi->type == I40E_VSI_MAIN)
14037                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14038
14039         /* assign it some queues */
14040         ret = i40e_alloc_rings(vsi);
14041         if (ret)
14042                 goto err_rings;
14043
14044         /* map all of the rings to the q_vectors */
14045         i40e_vsi_map_rings_to_vectors(vsi);
14046         return vsi;
14047
14048 err_rings:
14049         i40e_vsi_free_q_vectors(vsi);
14050         if (vsi->netdev_registered) {
14051                 vsi->netdev_registered = false;
14052                 unregister_netdev(vsi->netdev);
14053                 free_netdev(vsi->netdev);
14054                 vsi->netdev = NULL;
14055         }
14056         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14057 err_vsi:
14058         i40e_vsi_clear(vsi);
14059         return NULL;
14060 }
14061
14062 /**
14063  * i40e_vsi_setup - Set up a VSI by a given type
14064  * @pf: board private structure
14065  * @type: VSI type
14066  * @uplink_seid: the switch element to link to
14067  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14068  *
14069  * This allocates the sw VSI structure and its queue resources, then add a VSI
14070  * to the identified VEB.
14071  *
14072  * Returns pointer to the successfully allocated and configure VSI sw struct on
14073  * success, otherwise returns NULL on failure.
14074  **/
14075 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14076                                 u16 uplink_seid, u32 param1)
14077 {
14078         struct i40e_vsi *vsi = NULL;
14079         struct i40e_veb *veb = NULL;
14080         u16 alloc_queue_pairs;
14081         int ret, i;
14082         int v_idx;
14083
14084         /* The requested uplink_seid must be either
14085          *     - the PF's port seid
14086          *              no VEB is needed because this is the PF
14087          *              or this is a Flow Director special case VSI
14088          *     - seid of an existing VEB
14089          *     - seid of a VSI that owns an existing VEB
14090          *     - seid of a VSI that doesn't own a VEB
14091          *              a new VEB is created and the VSI becomes the owner
14092          *     - seid of the PF VSI, which is what creates the first VEB
14093          *              this is a special case of the previous
14094          *
14095          * Find which uplink_seid we were given and create a new VEB if needed
14096          */
14097         for (i = 0; i < I40E_MAX_VEB; i++) {
14098                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
14099                         veb = pf->veb[i];
14100                         break;
14101                 }
14102         }
14103
14104         if (!veb && uplink_seid != pf->mac_seid) {
14105
14106                 for (i = 0; i < pf->num_alloc_vsi; i++) {
14107                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
14108                                 vsi = pf->vsi[i];
14109                                 break;
14110                         }
14111                 }
14112                 if (!vsi) {
14113                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14114                                  uplink_seid);
14115                         return NULL;
14116                 }
14117
14118                 if (vsi->uplink_seid == pf->mac_seid)
14119                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14120                                              vsi->tc_config.enabled_tc);
14121                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14122                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14123                                              vsi->tc_config.enabled_tc);
14124                 if (veb) {
14125                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14126                                 dev_info(&vsi->back->pdev->dev,
14127                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
14128                                 return NULL;
14129                         }
14130                         /* We come up by default in VEPA mode if SRIOV is not
14131                          * already enabled, in which case we can't force VEPA
14132                          * mode.
14133                          */
14134                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
14135                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
14136                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
14137                         }
14138                         i40e_config_bridge_mode(veb);
14139                 }
14140                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
14141                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
14142                                 veb = pf->veb[i];
14143                 }
14144                 if (!veb) {
14145                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14146                         return NULL;
14147                 }
14148
14149                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14150                 uplink_seid = veb->seid;
14151         }
14152
14153         /* get vsi sw struct */
14154         v_idx = i40e_vsi_mem_alloc(pf, type);
14155         if (v_idx < 0)
14156                 goto err_alloc;
14157         vsi = pf->vsi[v_idx];
14158         if (!vsi)
14159                 goto err_alloc;
14160         vsi->type = type;
14161         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14162
14163         if (type == I40E_VSI_MAIN)
14164                 pf->lan_vsi = v_idx;
14165         else if (type == I40E_VSI_SRIOV)
14166                 vsi->vf_id = param1;
14167         /* assign it some queues */
14168         alloc_queue_pairs = vsi->alloc_queue_pairs *
14169                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14170
14171         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14172         if (ret < 0) {
14173                 dev_info(&pf->pdev->dev,
14174                          "failed to get tracking for %d queues for VSI %d err=%d\n",
14175                          alloc_queue_pairs, vsi->seid, ret);
14176                 goto err_vsi;
14177         }
14178         vsi->base_queue = ret;
14179
14180         /* get a VSI from the hardware */
14181         vsi->uplink_seid = uplink_seid;
14182         ret = i40e_add_vsi(vsi);
14183         if (ret)
14184                 goto err_vsi;
14185
14186         switch (vsi->type) {
14187         /* setup the netdev if needed */
14188         case I40E_VSI_MAIN:
14189         case I40E_VSI_VMDQ2:
14190                 ret = i40e_config_netdev(vsi);
14191                 if (ret)
14192                         goto err_netdev;
14193                 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14194                 if (ret)
14195                         goto err_netdev;
14196                 ret = register_netdev(vsi->netdev);
14197                 if (ret)
14198                         goto err_netdev;
14199                 vsi->netdev_registered = true;
14200                 netif_carrier_off(vsi->netdev);
14201 #ifdef CONFIG_I40E_DCB
14202                 /* Setup DCB netlink interface */
14203                 i40e_dcbnl_setup(vsi);
14204 #endif /* CONFIG_I40E_DCB */
14205                 fallthrough;
14206         case I40E_VSI_FDIR:
14207                 /* set up vectors and rings if needed */
14208                 ret = i40e_vsi_setup_vectors(vsi);
14209                 if (ret)
14210                         goto err_msix;
14211
14212                 ret = i40e_alloc_rings(vsi);
14213                 if (ret)
14214                         goto err_rings;
14215
14216                 /* map all of the rings to the q_vectors */
14217                 i40e_vsi_map_rings_to_vectors(vsi);
14218
14219                 i40e_vsi_reset_stats(vsi);
14220                 break;
14221         default:
14222                 /* no netdev or rings for the other VSI types */
14223                 break;
14224         }
14225
14226         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
14227             (vsi->type == I40E_VSI_VMDQ2)) {
14228                 ret = i40e_vsi_config_rss(vsi);
14229         }
14230         return vsi;
14231
14232 err_rings:
14233         i40e_vsi_free_q_vectors(vsi);
14234 err_msix:
14235         if (vsi->netdev_registered) {
14236                 vsi->netdev_registered = false;
14237                 unregister_netdev(vsi->netdev);
14238                 free_netdev(vsi->netdev);
14239                 vsi->netdev = NULL;
14240         }
14241 err_netdev:
14242         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14243 err_vsi:
14244         i40e_vsi_clear(vsi);
14245 err_alloc:
14246         return NULL;
14247 }
14248
14249 /**
14250  * i40e_veb_get_bw_info - Query VEB BW information
14251  * @veb: the veb to query
14252  *
14253  * Query the Tx scheduler BW configuration data for given VEB
14254  **/
14255 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14256 {
14257         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14258         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14259         struct i40e_pf *pf = veb->pf;
14260         struct i40e_hw *hw = &pf->hw;
14261         u32 tc_bw_max;
14262         int ret = 0;
14263         int i;
14264
14265         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14266                                                   &bw_data, NULL);
14267         if (ret) {
14268                 dev_info(&pf->pdev->dev,
14269                          "query veb bw config failed, err %s aq_err %s\n",
14270                          i40e_stat_str(&pf->hw, ret),
14271                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14272                 goto out;
14273         }
14274
14275         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14276                                                    &ets_data, NULL);
14277         if (ret) {
14278                 dev_info(&pf->pdev->dev,
14279                          "query veb bw ets config failed, err %s aq_err %s\n",
14280                          i40e_stat_str(&pf->hw, ret),
14281                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14282                 goto out;
14283         }
14284
14285         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14286         veb->bw_max_quanta = ets_data.tc_bw_max;
14287         veb->is_abs_credits = bw_data.absolute_credits_enable;
14288         veb->enabled_tc = ets_data.tc_valid_bits;
14289         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14290                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14291         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14292                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14293                 veb->bw_tc_limit_credits[i] =
14294                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
14295                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14296         }
14297
14298 out:
14299         return ret;
14300 }
14301
14302 /**
14303  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14304  * @pf: board private structure
14305  *
14306  * On error: returns error code (negative)
14307  * On success: returns vsi index in PF (positive)
14308  **/
14309 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14310 {
14311         int ret = -ENOENT;
14312         struct i40e_veb *veb;
14313         int i;
14314
14315         /* Need to protect the allocation of switch elements at the PF level */
14316         mutex_lock(&pf->switch_mutex);
14317
14318         /* VEB list may be fragmented if VEB creation/destruction has
14319          * been happening.  We can afford to do a quick scan to look
14320          * for any free slots in the list.
14321          *
14322          * find next empty veb slot, looping back around if necessary
14323          */
14324         i = 0;
14325         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14326                 i++;
14327         if (i >= I40E_MAX_VEB) {
14328                 ret = -ENOMEM;
14329                 goto err_alloc_veb;  /* out of VEB slots! */
14330         }
14331
14332         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14333         if (!veb) {
14334                 ret = -ENOMEM;
14335                 goto err_alloc_veb;
14336         }
14337         veb->pf = pf;
14338         veb->idx = i;
14339         veb->enabled_tc = 1;
14340
14341         pf->veb[i] = veb;
14342         ret = i;
14343 err_alloc_veb:
14344         mutex_unlock(&pf->switch_mutex);
14345         return ret;
14346 }
14347
14348 /**
14349  * i40e_switch_branch_release - Delete a branch of the switch tree
14350  * @branch: where to start deleting
14351  *
14352  * This uses recursion to find the tips of the branch to be
14353  * removed, deleting until we get back to and can delete this VEB.
14354  **/
14355 static void i40e_switch_branch_release(struct i40e_veb *branch)
14356 {
14357         struct i40e_pf *pf = branch->pf;
14358         u16 branch_seid = branch->seid;
14359         u16 veb_idx = branch->idx;
14360         int i;
14361
14362         /* release any VEBs on this VEB - RECURSION */
14363         for (i = 0; i < I40E_MAX_VEB; i++) {
14364                 if (!pf->veb[i])
14365                         continue;
14366                 if (pf->veb[i]->uplink_seid == branch->seid)
14367                         i40e_switch_branch_release(pf->veb[i]);
14368         }
14369
14370         /* Release the VSIs on this VEB, but not the owner VSI.
14371          *
14372          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14373          *       the VEB itself, so don't use (*branch) after this loop.
14374          */
14375         for (i = 0; i < pf->num_alloc_vsi; i++) {
14376                 if (!pf->vsi[i])
14377                         continue;
14378                 if (pf->vsi[i]->uplink_seid == branch_seid &&
14379                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14380                         i40e_vsi_release(pf->vsi[i]);
14381                 }
14382         }
14383
14384         /* There's one corner case where the VEB might not have been
14385          * removed, so double check it here and remove it if needed.
14386          * This case happens if the veb was created from the debugfs
14387          * commands and no VSIs were added to it.
14388          */
14389         if (pf->veb[veb_idx])
14390                 i40e_veb_release(pf->veb[veb_idx]);
14391 }
14392
14393 /**
14394  * i40e_veb_clear - remove veb struct
14395  * @veb: the veb to remove
14396  **/
14397 static void i40e_veb_clear(struct i40e_veb *veb)
14398 {
14399         if (!veb)
14400                 return;
14401
14402         if (veb->pf) {
14403                 struct i40e_pf *pf = veb->pf;
14404
14405                 mutex_lock(&pf->switch_mutex);
14406                 if (pf->veb[veb->idx] == veb)
14407                         pf->veb[veb->idx] = NULL;
14408                 mutex_unlock(&pf->switch_mutex);
14409         }
14410
14411         kfree(veb);
14412 }
14413
14414 /**
14415  * i40e_veb_release - Delete a VEB and free its resources
14416  * @veb: the VEB being removed
14417  **/
14418 void i40e_veb_release(struct i40e_veb *veb)
14419 {
14420         struct i40e_vsi *vsi = NULL;
14421         struct i40e_pf *pf;
14422         int i, n = 0;
14423
14424         pf = veb->pf;
14425
14426         /* find the remaining VSI and check for extras */
14427         for (i = 0; i < pf->num_alloc_vsi; i++) {
14428                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14429                         n++;
14430                         vsi = pf->vsi[i];
14431                 }
14432         }
14433         if (n != 1) {
14434                 dev_info(&pf->pdev->dev,
14435                          "can't remove VEB %d with %d VSIs left\n",
14436                          veb->seid, n);
14437                 return;
14438         }
14439
14440         /* move the remaining VSI to uplink veb */
14441         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14442         if (veb->uplink_seid) {
14443                 vsi->uplink_seid = veb->uplink_seid;
14444                 if (veb->uplink_seid == pf->mac_seid)
14445                         vsi->veb_idx = I40E_NO_VEB;
14446                 else
14447                         vsi->veb_idx = veb->veb_idx;
14448         } else {
14449                 /* floating VEB */
14450                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14451                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14452         }
14453
14454         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14455         i40e_veb_clear(veb);
14456 }
14457
14458 /**
14459  * i40e_add_veb - create the VEB in the switch
14460  * @veb: the VEB to be instantiated
14461  * @vsi: the controlling VSI
14462  **/
14463 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14464 {
14465         struct i40e_pf *pf = veb->pf;
14466         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14467         int ret;
14468
14469         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14470                               veb->enabled_tc, false,
14471                               &veb->seid, enable_stats, NULL);
14472
14473         /* get a VEB from the hardware */
14474         if (ret) {
14475                 dev_info(&pf->pdev->dev,
14476                          "couldn't add VEB, err %s aq_err %s\n",
14477                          i40e_stat_str(&pf->hw, ret),
14478                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14479                 return -EPERM;
14480         }
14481
14482         /* get statistics counter */
14483         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14484                                          &veb->stats_idx, NULL, NULL, NULL);
14485         if (ret) {
14486                 dev_info(&pf->pdev->dev,
14487                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
14488                          i40e_stat_str(&pf->hw, ret),
14489                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14490                 return -EPERM;
14491         }
14492         ret = i40e_veb_get_bw_info(veb);
14493         if (ret) {
14494                 dev_info(&pf->pdev->dev,
14495                          "couldn't get VEB bw info, err %s aq_err %s\n",
14496                          i40e_stat_str(&pf->hw, ret),
14497                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14498                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14499                 return -ENOENT;
14500         }
14501
14502         vsi->uplink_seid = veb->seid;
14503         vsi->veb_idx = veb->idx;
14504         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14505
14506         return 0;
14507 }
14508
14509 /**
14510  * i40e_veb_setup - Set up a VEB
14511  * @pf: board private structure
14512  * @flags: VEB setup flags
14513  * @uplink_seid: the switch element to link to
14514  * @vsi_seid: the initial VSI seid
14515  * @enabled_tc: Enabled TC bit-map
14516  *
14517  * This allocates the sw VEB structure and links it into the switch
14518  * It is possible and legal for this to be a duplicate of an already
14519  * existing VEB.  It is also possible for both uplink and vsi seids
14520  * to be zero, in order to create a floating VEB.
14521  *
14522  * Returns pointer to the successfully allocated VEB sw struct on
14523  * success, otherwise returns NULL on failure.
14524  **/
14525 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14526                                 u16 uplink_seid, u16 vsi_seid,
14527                                 u8 enabled_tc)
14528 {
14529         struct i40e_veb *veb, *uplink_veb = NULL;
14530         int vsi_idx, veb_idx;
14531         int ret;
14532
14533         /* if one seid is 0, the other must be 0 to create a floating relay */
14534         if ((uplink_seid == 0 || vsi_seid == 0) &&
14535             (uplink_seid + vsi_seid != 0)) {
14536                 dev_info(&pf->pdev->dev,
14537                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14538                          uplink_seid, vsi_seid);
14539                 return NULL;
14540         }
14541
14542         /* make sure there is such a vsi and uplink */
14543         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14544                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14545                         break;
14546         if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14547                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14548                          vsi_seid);
14549                 return NULL;
14550         }
14551
14552         if (uplink_seid && uplink_seid != pf->mac_seid) {
14553                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14554                         if (pf->veb[veb_idx] &&
14555                             pf->veb[veb_idx]->seid == uplink_seid) {
14556                                 uplink_veb = pf->veb[veb_idx];
14557                                 break;
14558                         }
14559                 }
14560                 if (!uplink_veb) {
14561                         dev_info(&pf->pdev->dev,
14562                                  "uplink seid %d not found\n", uplink_seid);
14563                         return NULL;
14564                 }
14565         }
14566
14567         /* get veb sw struct */
14568         veb_idx = i40e_veb_mem_alloc(pf);
14569         if (veb_idx < 0)
14570                 goto err_alloc;
14571         veb = pf->veb[veb_idx];
14572         veb->flags = flags;
14573         veb->uplink_seid = uplink_seid;
14574         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14575         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14576
14577         /* create the VEB in the switch */
14578         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14579         if (ret)
14580                 goto err_veb;
14581         if (vsi_idx == pf->lan_vsi)
14582                 pf->lan_veb = veb->idx;
14583
14584         return veb;
14585
14586 err_veb:
14587         i40e_veb_clear(veb);
14588 err_alloc:
14589         return NULL;
14590 }
14591
14592 /**
14593  * i40e_setup_pf_switch_element - set PF vars based on switch type
14594  * @pf: board private structure
14595  * @ele: element we are building info from
14596  * @num_reported: total number of elements
14597  * @printconfig: should we print the contents
14598  *
14599  * helper function to assist in extracting a few useful SEID values.
14600  **/
14601 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14602                                 struct i40e_aqc_switch_config_element_resp *ele,
14603                                 u16 num_reported, bool printconfig)
14604 {
14605         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14606         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14607         u8 element_type = ele->element_type;
14608         u16 seid = le16_to_cpu(ele->seid);
14609
14610         if (printconfig)
14611                 dev_info(&pf->pdev->dev,
14612                          "type=%d seid=%d uplink=%d downlink=%d\n",
14613                          element_type, seid, uplink_seid, downlink_seid);
14614
14615         switch (element_type) {
14616         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14617                 pf->mac_seid = seid;
14618                 break;
14619         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14620                 /* Main VEB? */
14621                 if (uplink_seid != pf->mac_seid)
14622                         break;
14623                 if (pf->lan_veb >= I40E_MAX_VEB) {
14624                         int v;
14625
14626                         /* find existing or else empty VEB */
14627                         for (v = 0; v < I40E_MAX_VEB; v++) {
14628                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14629                                         pf->lan_veb = v;
14630                                         break;
14631                                 }
14632                         }
14633                         if (pf->lan_veb >= I40E_MAX_VEB) {
14634                                 v = i40e_veb_mem_alloc(pf);
14635                                 if (v < 0)
14636                                         break;
14637                                 pf->lan_veb = v;
14638                         }
14639                 }
14640                 if (pf->lan_veb >= I40E_MAX_VEB)
14641                         break;
14642
14643                 pf->veb[pf->lan_veb]->seid = seid;
14644                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14645                 pf->veb[pf->lan_veb]->pf = pf;
14646                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14647                 break;
14648         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14649                 if (num_reported != 1)
14650                         break;
14651                 /* This is immediately after a reset so we can assume this is
14652                  * the PF's VSI
14653                  */
14654                 pf->mac_seid = uplink_seid;
14655                 pf->pf_seid = downlink_seid;
14656                 pf->main_vsi_seid = seid;
14657                 if (printconfig)
14658                         dev_info(&pf->pdev->dev,
14659                                  "pf_seid=%d main_vsi_seid=%d\n",
14660                                  pf->pf_seid, pf->main_vsi_seid);
14661                 break;
14662         case I40E_SWITCH_ELEMENT_TYPE_PF:
14663         case I40E_SWITCH_ELEMENT_TYPE_VF:
14664         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14665         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14666         case I40E_SWITCH_ELEMENT_TYPE_PE:
14667         case I40E_SWITCH_ELEMENT_TYPE_PA:
14668                 /* ignore these for now */
14669                 break;
14670         default:
14671                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14672                          element_type, seid);
14673                 break;
14674         }
14675 }
14676
14677 /**
14678  * i40e_fetch_switch_configuration - Get switch config from firmware
14679  * @pf: board private structure
14680  * @printconfig: should we print the contents
14681  *
14682  * Get the current switch configuration from the device and
14683  * extract a few useful SEID values.
14684  **/
14685 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14686 {
14687         struct i40e_aqc_get_switch_config_resp *sw_config;
14688         u16 next_seid = 0;
14689         int ret = 0;
14690         u8 *aq_buf;
14691         int i;
14692
14693         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14694         if (!aq_buf)
14695                 return -ENOMEM;
14696
14697         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14698         do {
14699                 u16 num_reported, num_total;
14700
14701                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14702                                                 I40E_AQ_LARGE_BUF,
14703                                                 &next_seid, NULL);
14704                 if (ret) {
14705                         dev_info(&pf->pdev->dev,
14706                                  "get switch config failed err %s aq_err %s\n",
14707                                  i40e_stat_str(&pf->hw, ret),
14708                                  i40e_aq_str(&pf->hw,
14709                                              pf->hw.aq.asq_last_status));
14710                         kfree(aq_buf);
14711                         return -ENOENT;
14712                 }
14713
14714                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14715                 num_total = le16_to_cpu(sw_config->header.num_total);
14716
14717                 if (printconfig)
14718                         dev_info(&pf->pdev->dev,
14719                                  "header: %d reported %d total\n",
14720                                  num_reported, num_total);
14721
14722                 for (i = 0; i < num_reported; i++) {
14723                         struct i40e_aqc_switch_config_element_resp *ele =
14724                                 &sw_config->element[i];
14725
14726                         i40e_setup_pf_switch_element(pf, ele, num_reported,
14727                                                      printconfig);
14728                 }
14729         } while (next_seid != 0);
14730
14731         kfree(aq_buf);
14732         return ret;
14733 }
14734
14735 /**
14736  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14737  * @pf: board private structure
14738  * @reinit: if the Main VSI needs to re-initialized.
14739  * @lock_acquired: indicates whether or not the lock has been acquired
14740  *
14741  * Returns 0 on success, negative value on failure
14742  **/
14743 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
14744 {
14745         u16 flags = 0;
14746         int ret;
14747
14748         /* find out what's out there already */
14749         ret = i40e_fetch_switch_configuration(pf, false);
14750         if (ret) {
14751                 dev_info(&pf->pdev->dev,
14752                          "couldn't fetch switch config, err %s aq_err %s\n",
14753                          i40e_stat_str(&pf->hw, ret),
14754                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14755                 return ret;
14756         }
14757         i40e_pf_reset_stats(pf);
14758
14759         /* set the switch config bit for the whole device to
14760          * support limited promisc or true promisc
14761          * when user requests promisc. The default is limited
14762          * promisc.
14763         */
14764
14765         if ((pf->hw.pf_id == 0) &&
14766             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14767                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14768                 pf->last_sw_conf_flags = flags;
14769         }
14770
14771         if (pf->hw.pf_id == 0) {
14772                 u16 valid_flags;
14773
14774                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14775                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14776                                                 NULL);
14777                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14778                         dev_info(&pf->pdev->dev,
14779                                  "couldn't set switch config bits, err %s aq_err %s\n",
14780                                  i40e_stat_str(&pf->hw, ret),
14781                                  i40e_aq_str(&pf->hw,
14782                                              pf->hw.aq.asq_last_status));
14783                         /* not a fatal problem, just keep going */
14784                 }
14785                 pf->last_sw_conf_valid_flags = valid_flags;
14786         }
14787
14788         /* first time setup */
14789         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14790                 struct i40e_vsi *vsi = NULL;
14791                 u16 uplink_seid;
14792
14793                 /* Set up the PF VSI associated with the PF's main VSI
14794                  * that is already in the HW switch
14795                  */
14796                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14797                         uplink_seid = pf->veb[pf->lan_veb]->seid;
14798                 else
14799                         uplink_seid = pf->mac_seid;
14800                 if (pf->lan_vsi == I40E_NO_VSI)
14801                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14802                 else if (reinit)
14803                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14804                 if (!vsi) {
14805                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14806                         i40e_cloud_filter_exit(pf);
14807                         i40e_fdir_teardown(pf);
14808                         return -EAGAIN;
14809                 }
14810         } else {
14811                 /* force a reset of TC and queue layout configurations */
14812                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14813
14814                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14815                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14816                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14817         }
14818         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14819
14820         i40e_fdir_sb_setup(pf);
14821
14822         /* Setup static PF queue filter control settings */
14823         ret = i40e_setup_pf_filter_control(pf);
14824         if (ret) {
14825                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14826                          ret);
14827                 /* Failure here should not stop continuing other steps */
14828         }
14829
14830         /* enable RSS in the HW, even for only one queue, as the stack can use
14831          * the hash
14832          */
14833         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14834                 i40e_pf_config_rss(pf);
14835
14836         /* fill in link information and enable LSE reporting */
14837         i40e_link_event(pf);
14838
14839         /* Initialize user-specific link properties */
14840         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14841                                   I40E_AQ_AN_COMPLETED) ? true : false);
14842
14843         i40e_ptp_init(pf);
14844
14845         if (!lock_acquired)
14846                 rtnl_lock();
14847
14848         /* repopulate tunnel port filters */
14849         udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
14850
14851         if (!lock_acquired)
14852                 rtnl_unlock();
14853
14854         return ret;
14855 }
14856
14857 /**
14858  * i40e_determine_queue_usage - Work out queue distribution
14859  * @pf: board private structure
14860  **/
14861 static void i40e_determine_queue_usage(struct i40e_pf *pf)
14862 {
14863         int queues_left;
14864         int q_max;
14865
14866         pf->num_lan_qps = 0;
14867
14868         /* Find the max queues to be put into basic use.  We'll always be
14869          * using TC0, whether or not DCB is running, and TC0 will get the
14870          * big RSS set.
14871          */
14872         queues_left = pf->hw.func_caps.num_tx_qp;
14873
14874         if ((queues_left == 1) ||
14875             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14876                 /* one qp for PF, no queues for anything else */
14877                 queues_left = 0;
14878                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14879
14880                 /* make sure all the fancies are disabled */
14881                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14882                                I40E_FLAG_IWARP_ENABLED  |
14883                                I40E_FLAG_FD_SB_ENABLED  |
14884                                I40E_FLAG_FD_ATR_ENABLED |
14885                                I40E_FLAG_DCB_CAPABLE    |
14886                                I40E_FLAG_DCB_ENABLED    |
14887                                I40E_FLAG_SRIOV_ENABLED  |
14888                                I40E_FLAG_VMDQ_ENABLED);
14889                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14890         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14891                                   I40E_FLAG_FD_SB_ENABLED |
14892                                   I40E_FLAG_FD_ATR_ENABLED |
14893                                   I40E_FLAG_DCB_CAPABLE))) {
14894                 /* one qp for PF */
14895                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14896                 queues_left -= pf->num_lan_qps;
14897
14898                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14899                                I40E_FLAG_IWARP_ENABLED  |
14900                                I40E_FLAG_FD_SB_ENABLED  |
14901                                I40E_FLAG_FD_ATR_ENABLED |
14902                                I40E_FLAG_DCB_ENABLED    |
14903                                I40E_FLAG_VMDQ_ENABLED);
14904                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14905         } else {
14906                 /* Not enough queues for all TCs */
14907                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14908                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14909                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14910                                         I40E_FLAG_DCB_ENABLED);
14911                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14912                 }
14913
14914                 /* limit lan qps to the smaller of qps, cpus or msix */
14915                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14916                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14917                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14918                 pf->num_lan_qps = q_max;
14919
14920                 queues_left -= pf->num_lan_qps;
14921         }
14922
14923         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14924                 if (queues_left > 1) {
14925                         queues_left -= 1; /* save 1 queue for FD */
14926                 } else {
14927                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14928                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14929                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14930                 }
14931         }
14932
14933         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14934             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14935                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14936                                         (queues_left / pf->num_vf_qps));
14937                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14938         }
14939
14940         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14941             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14942                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14943                                           (queues_left / pf->num_vmdq_qps));
14944                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14945         }
14946
14947         pf->queues_left = queues_left;
14948         dev_dbg(&pf->pdev->dev,
14949                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14950                 pf->hw.func_caps.num_tx_qp,
14951                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14952                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14953                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14954                 queues_left);
14955 }
14956
14957 /**
14958  * i40e_setup_pf_filter_control - Setup PF static filter control
14959  * @pf: PF to be setup
14960  *
14961  * i40e_setup_pf_filter_control sets up a PF's initial filter control
14962  * settings. If PE/FCoE are enabled then it will also set the per PF
14963  * based filter sizes required for them. It also enables Flow director,
14964  * ethertype and macvlan type filter settings for the pf.
14965  *
14966  * Returns 0 on success, negative on failure
14967  **/
14968 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14969 {
14970         struct i40e_filter_control_settings *settings = &pf->filter_settings;
14971
14972         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14973
14974         /* Flow Director is enabled */
14975         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14976                 settings->enable_fdir = true;
14977
14978         /* Ethtype and MACVLAN filters enabled for PF */
14979         settings->enable_ethtype = true;
14980         settings->enable_macvlan = true;
14981
14982         if (i40e_set_filter_control(&pf->hw, settings))
14983                 return -ENOENT;
14984
14985         return 0;
14986 }
14987
14988 #define INFO_STRING_LEN 255
14989 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
14990 static void i40e_print_features(struct i40e_pf *pf)
14991 {
14992         struct i40e_hw *hw = &pf->hw;
14993         char *buf;
14994         int i;
14995
14996         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14997         if (!buf)
14998                 return;
14999
15000         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15001 #ifdef CONFIG_PCI_IOV
15002         i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15003 #endif
15004         i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15005                       pf->hw.func_caps.num_vsis,
15006                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
15007         if (pf->flags & I40E_FLAG_RSS_ENABLED)
15008                 i += scnprintf(&buf[i], REMAIN(i), " RSS");
15009         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
15010                 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15011         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
15012                 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15013                 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15014         }
15015         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
15016                 i += scnprintf(&buf[i], REMAIN(i), " DCB");
15017         i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15018         i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15019         if (pf->flags & I40E_FLAG_PTP)
15020                 i += scnprintf(&buf[i], REMAIN(i), " PTP");
15021         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
15022                 i += scnprintf(&buf[i], REMAIN(i), " VEB");
15023         else
15024                 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15025
15026         dev_info(&pf->pdev->dev, "%s\n", buf);
15027         kfree(buf);
15028         WARN_ON(i > INFO_STRING_LEN);
15029 }
15030
15031 /**
15032  * i40e_get_platform_mac_addr - get platform-specific MAC address
15033  * @pdev: PCI device information struct
15034  * @pf: board private structure
15035  *
15036  * Look up the MAC address for the device. First we'll try
15037  * eth_platform_get_mac_address, which will check Open Firmware, or arch
15038  * specific fallback. Otherwise, we'll default to the stored value in
15039  * firmware.
15040  **/
15041 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15042 {
15043         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15044                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15045 }
15046
15047 /**
15048  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15049  * @fec_cfg: FEC option to set in flags
15050  * @flags: ptr to flags in which we set FEC option
15051  **/
15052 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
15053 {
15054         if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
15055                 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
15056         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15057             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15058                 *flags |= I40E_FLAG_RS_FEC;
15059                 *flags &= ~I40E_FLAG_BASE_R_FEC;
15060         }
15061         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15062             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15063                 *flags |= I40E_FLAG_BASE_R_FEC;
15064                 *flags &= ~I40E_FLAG_RS_FEC;
15065         }
15066         if (fec_cfg == 0)
15067                 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
15068 }
15069
15070 /**
15071  * i40e_check_recovery_mode - check if we are running transition firmware
15072  * @pf: board private structure
15073  *
15074  * Check registers indicating the firmware runs in recovery mode. Sets the
15075  * appropriate driver state.
15076  *
15077  * Returns true if the recovery mode was detected, false otherwise
15078  **/
15079 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15080 {
15081         u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15082
15083         if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15084                 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15085                 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15086                 set_bit(__I40E_RECOVERY_MODE, pf->state);
15087
15088                 return true;
15089         }
15090         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15091                 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15092
15093         return false;
15094 }
15095
15096 /**
15097  * i40e_pf_loop_reset - perform reset in a loop.
15098  * @pf: board private structure
15099  *
15100  * This function is useful when a NIC is about to enter recovery mode.
15101  * When a NIC's internal data structures are corrupted the NIC's
15102  * firmware is going to enter recovery mode.
15103  * Right after a POR it takes about 7 minutes for firmware to enter
15104  * recovery mode. Until that time a NIC is in some kind of intermediate
15105  * state. After that time period the NIC almost surely enters
15106  * recovery mode. The only way for a driver to detect intermediate
15107  * state is to issue a series of pf-resets and check a return value.
15108  * If a PF reset returns success then the firmware could be in recovery
15109  * mode so the caller of this code needs to check for recovery mode
15110  * if this function returns success. There is a little chance that
15111  * firmware will hang in intermediate state forever.
15112  * Since waiting 7 minutes is quite a lot of time this function waits
15113  * 10 seconds and then gives up by returning an error.
15114  *
15115  * Return 0 on success, negative on failure.
15116  **/
15117 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf)
15118 {
15119         /* wait max 10 seconds for PF reset to succeed */
15120         const unsigned long time_end = jiffies + 10 * HZ;
15121
15122         struct i40e_hw *hw = &pf->hw;
15123         i40e_status ret;
15124
15125         ret = i40e_pf_reset(hw);
15126         while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
15127                 usleep_range(10000, 20000);
15128                 ret = i40e_pf_reset(hw);
15129         }
15130
15131         if (ret == I40E_SUCCESS)
15132                 pf->pfr_count++;
15133         else
15134                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15135
15136         return ret;
15137 }
15138
15139 /**
15140  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15141  * @pf: board private structure
15142  *
15143  * Check FW registers to determine if FW issued unexpected EMP Reset.
15144  * Every time when unexpected EMP Reset occurs the FW increments
15145  * a counter of unexpected EMP Resets. When the counter reaches 10
15146  * the FW should enter the Recovery mode
15147  *
15148  * Returns true if FW issued unexpected EMP Reset
15149  **/
15150 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15151 {
15152         const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15153                            I40E_GL_FWSTS_FWS1B_MASK;
15154         return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15155                (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15156 }
15157
15158 /**
15159  * i40e_handle_resets - handle EMP resets and PF resets
15160  * @pf: board private structure
15161  *
15162  * Handle both EMP resets and PF resets and conclude whether there are
15163  * any issues regarding these resets. If there are any issues then
15164  * generate log entry.
15165  *
15166  * Return 0 if NIC is healthy or negative value when there are issues
15167  * with resets
15168  **/
15169 static i40e_status i40e_handle_resets(struct i40e_pf *pf)
15170 {
15171         const i40e_status pfr = i40e_pf_loop_reset(pf);
15172         const bool is_empr = i40e_check_fw_empr(pf);
15173
15174         if (is_empr || pfr != I40E_SUCCESS)
15175                 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15176
15177         return is_empr ? I40E_ERR_RESET_FAILED : pfr;
15178 }
15179
15180 /**
15181  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15182  * @pf: board private structure
15183  * @hw: ptr to the hardware info
15184  *
15185  * This function does a minimal setup of all subsystems needed for running
15186  * recovery mode.
15187  *
15188  * Returns 0 on success, negative on failure
15189  **/
15190 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15191 {
15192         struct i40e_vsi *vsi;
15193         int err;
15194         int v_idx;
15195
15196         pci_save_state(pf->pdev);
15197
15198         /* set up periodic task facility */
15199         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15200         pf->service_timer_period = HZ;
15201
15202         INIT_WORK(&pf->service_task, i40e_service_task);
15203         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15204
15205         err = i40e_init_interrupt_scheme(pf);
15206         if (err)
15207                 goto err_switch_setup;
15208
15209         /* The number of VSIs reported by the FW is the minimum guaranteed
15210          * to us; HW supports far more and we share the remaining pool with
15211          * the other PFs. We allocate space for more than the guarantee with
15212          * the understanding that we might not get them all later.
15213          */
15214         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15215                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15216         else
15217                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15218
15219         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15220         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15221                           GFP_KERNEL);
15222         if (!pf->vsi) {
15223                 err = -ENOMEM;
15224                 goto err_switch_setup;
15225         }
15226
15227         /* We allocate one VSI which is needed as absolute minimum
15228          * in order to register the netdev
15229          */
15230         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15231         if (v_idx < 0) {
15232                 err = v_idx;
15233                 goto err_switch_setup;
15234         }
15235         pf->lan_vsi = v_idx;
15236         vsi = pf->vsi[v_idx];
15237         if (!vsi) {
15238                 err = -EFAULT;
15239                 goto err_switch_setup;
15240         }
15241         vsi->alloc_queue_pairs = 1;
15242         err = i40e_config_netdev(vsi);
15243         if (err)
15244                 goto err_switch_setup;
15245         err = register_netdev(vsi->netdev);
15246         if (err)
15247                 goto err_switch_setup;
15248         vsi->netdev_registered = true;
15249         i40e_dbg_pf_init(pf);
15250
15251         err = i40e_setup_misc_vector_for_recovery_mode(pf);
15252         if (err)
15253                 goto err_switch_setup;
15254
15255         /* tell the firmware that we're starting */
15256         i40e_send_version(pf);
15257
15258         /* since everything's happy, start the service_task timer */
15259         mod_timer(&pf->service_timer,
15260                   round_jiffies(jiffies + pf->service_timer_period));
15261
15262         return 0;
15263
15264 err_switch_setup:
15265         i40e_reset_interrupt_capability(pf);
15266         del_timer_sync(&pf->service_timer);
15267         i40e_shutdown_adminq(hw);
15268         iounmap(hw->hw_addr);
15269         pci_disable_pcie_error_reporting(pf->pdev);
15270         pci_release_mem_regions(pf->pdev);
15271         pci_disable_device(pf->pdev);
15272         kfree(pf);
15273
15274         return err;
15275 }
15276
15277 /**
15278  * i40e_set_subsystem_device_id - set subsystem device id
15279  * @hw: pointer to the hardware info
15280  *
15281  * Set PCI subsystem device id either from a pci_dev structure or
15282  * a specific FW register.
15283  **/
15284 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15285 {
15286         struct pci_dev *pdev = ((struct i40e_pf *)hw->back)->pdev;
15287
15288         hw->subsystem_device_id = pdev->subsystem_device ?
15289                 pdev->subsystem_device :
15290                 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15291 }
15292
15293 /**
15294  * i40e_probe - Device initialization routine
15295  * @pdev: PCI device information struct
15296  * @ent: entry in i40e_pci_tbl
15297  *
15298  * i40e_probe initializes a PF identified by a pci_dev structure.
15299  * The OS initialization, configuring of the PF private structure,
15300  * and a hardware reset occur.
15301  *
15302  * Returns 0 on success, negative on failure
15303  **/
15304 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15305 {
15306         struct i40e_aq_get_phy_abilities_resp abilities;
15307 #ifdef CONFIG_I40E_DCB
15308         enum i40e_get_fw_lldp_status_resp lldp_status;
15309         i40e_status status;
15310 #endif /* CONFIG_I40E_DCB */
15311         struct i40e_pf *pf;
15312         struct i40e_hw *hw;
15313         static u16 pfs_found;
15314         u16 wol_nvm_bits;
15315         u16 link_status;
15316         int err;
15317         u32 val;
15318         u32 i;
15319
15320         err = pci_enable_device_mem(pdev);
15321         if (err)
15322                 return err;
15323
15324         /* set up for high or low dma */
15325         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15326         if (err) {
15327                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
15328                 if (err) {
15329                         dev_err(&pdev->dev,
15330                                 "DMA configuration failed: 0x%x\n", err);
15331                         goto err_dma;
15332                 }
15333         }
15334
15335         /* set up pci connections */
15336         err = pci_request_mem_regions(pdev, i40e_driver_name);
15337         if (err) {
15338                 dev_info(&pdev->dev,
15339                          "pci_request_selected_regions failed %d\n", err);
15340                 goto err_pci_reg;
15341         }
15342
15343         pci_enable_pcie_error_reporting(pdev);
15344         pci_set_master(pdev);
15345
15346         /* Now that we have a PCI connection, we need to do the
15347          * low level device setup.  This is primarily setting up
15348          * the Admin Queue structures and then querying for the
15349          * device's current profile information.
15350          */
15351         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15352         if (!pf) {
15353                 err = -ENOMEM;
15354                 goto err_pf_alloc;
15355         }
15356         pf->next_vsi = 0;
15357         pf->pdev = pdev;
15358         set_bit(__I40E_DOWN, pf->state);
15359
15360         hw = &pf->hw;
15361         hw->back = pf;
15362
15363         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15364                                 I40E_MAX_CSR_SPACE);
15365         /* We believe that the highest register to read is
15366          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15367          * is not less than that before mapping to prevent a
15368          * kernel panic.
15369          */
15370         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15371                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15372                         pf->ioremap_len);
15373                 err = -ENOMEM;
15374                 goto err_ioremap;
15375         }
15376         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15377         if (!hw->hw_addr) {
15378                 err = -EIO;
15379                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15380                          (unsigned int)pci_resource_start(pdev, 0),
15381                          pf->ioremap_len, err);
15382                 goto err_ioremap;
15383         }
15384         hw->vendor_id = pdev->vendor;
15385         hw->device_id = pdev->device;
15386         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15387         hw->subsystem_vendor_id = pdev->subsystem_vendor;
15388         i40e_set_subsystem_device_id(hw);
15389         hw->bus.device = PCI_SLOT(pdev->devfn);
15390         hw->bus.func = PCI_FUNC(pdev->devfn);
15391         hw->bus.bus_id = pdev->bus->number;
15392         pf->instance = pfs_found;
15393
15394         /* Select something other than the 802.1ad ethertype for the
15395          * switch to use internally and drop on ingress.
15396          */
15397         hw->switch_tag = 0xffff;
15398         hw->first_tag = ETH_P_8021AD;
15399         hw->second_tag = ETH_P_8021Q;
15400
15401         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15402         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15403         INIT_LIST_HEAD(&pf->ddp_old_prof);
15404
15405         /* set up the locks for the AQ, do this only once in probe
15406          * and destroy them only once in remove
15407          */
15408         mutex_init(&hw->aq.asq_mutex);
15409         mutex_init(&hw->aq.arq_mutex);
15410
15411         pf->msg_enable = netif_msg_init(debug,
15412                                         NETIF_MSG_DRV |
15413                                         NETIF_MSG_PROBE |
15414                                         NETIF_MSG_LINK);
15415         if (debug < -1)
15416                 pf->hw.debug_mask = debug;
15417
15418         /* do a special CORER for clearing PXE mode once at init */
15419         if (hw->revision_id == 0 &&
15420             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15421                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15422                 i40e_flush(hw);
15423                 msleep(200);
15424                 pf->corer_count++;
15425
15426                 i40e_clear_pxe_mode(hw);
15427         }
15428
15429         /* Reset here to make sure all is clean and to define PF 'n' */
15430         i40e_clear_hw(hw);
15431
15432         err = i40e_set_mac_type(hw);
15433         if (err) {
15434                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15435                          err);
15436                 goto err_pf_reset;
15437         }
15438
15439         err = i40e_handle_resets(pf);
15440         if (err)
15441                 goto err_pf_reset;
15442
15443         i40e_check_recovery_mode(pf);
15444
15445         if (is_kdump_kernel()) {
15446                 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15447                 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15448         } else {
15449                 hw->aq.num_arq_entries = I40E_AQ_LEN;
15450                 hw->aq.num_asq_entries = I40E_AQ_LEN;
15451         }
15452         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15453         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15454         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15455
15456         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15457                  "%s-%s:misc",
15458                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15459
15460         err = i40e_init_shared_code(hw);
15461         if (err) {
15462                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15463                          err);
15464                 goto err_pf_reset;
15465         }
15466
15467         /* set up a default setting for link flow control */
15468         pf->hw.fc.requested_mode = I40E_FC_NONE;
15469
15470         err = i40e_init_adminq(hw);
15471         if (err) {
15472                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
15473                         dev_info(&pdev->dev,
15474                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15475                                  hw->aq.api_maj_ver,
15476                                  hw->aq.api_min_ver,
15477                                  I40E_FW_API_VERSION_MAJOR,
15478                                  I40E_FW_MINOR_VERSION(hw));
15479                 else
15480                         dev_info(&pdev->dev,
15481                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15482
15483                 goto err_pf_reset;
15484         }
15485         i40e_get_oem_version(hw);
15486
15487         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15488         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15489                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15490                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
15491                  i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15492                  hw->subsystem_vendor_id, hw->subsystem_device_id);
15493
15494         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15495             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15496                 dev_dbg(&pdev->dev,
15497                         "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15498                          hw->aq.api_maj_ver,
15499                          hw->aq.api_min_ver,
15500                          I40E_FW_API_VERSION_MAJOR,
15501                          I40E_FW_MINOR_VERSION(hw));
15502         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15503                 dev_info(&pdev->dev,
15504                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15505                          hw->aq.api_maj_ver,
15506                          hw->aq.api_min_ver,
15507                          I40E_FW_API_VERSION_MAJOR,
15508                          I40E_FW_MINOR_VERSION(hw));
15509
15510         i40e_verify_eeprom(pf);
15511
15512         /* Rev 0 hardware was never productized */
15513         if (hw->revision_id < 1)
15514                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15515
15516         i40e_clear_pxe_mode(hw);
15517
15518         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15519         if (err)
15520                 goto err_adminq_setup;
15521
15522         err = i40e_sw_init(pf);
15523         if (err) {
15524                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15525                 goto err_sw_init;
15526         }
15527
15528         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15529                 return i40e_init_recovery_mode(pf, hw);
15530
15531         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15532                                 hw->func_caps.num_rx_qp, 0, 0);
15533         if (err) {
15534                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15535                 goto err_init_lan_hmc;
15536         }
15537
15538         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15539         if (err) {
15540                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15541                 err = -ENOENT;
15542                 goto err_configure_lan_hmc;
15543         }
15544
15545         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15546          * Ignore error return codes because if it was already disabled via
15547          * hardware settings this will fail
15548          */
15549         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15550                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15551                 i40e_aq_stop_lldp(hw, true, false, NULL);
15552         }
15553
15554         /* allow a platform config to override the HW addr */
15555         i40e_get_platform_mac_addr(pdev, pf);
15556
15557         if (!is_valid_ether_addr(hw->mac.addr)) {
15558                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15559                 err = -EIO;
15560                 goto err_mac_addr;
15561         }
15562         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15563         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15564         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15565         if (is_valid_ether_addr(hw->mac.port_addr))
15566                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15567
15568         i40e_ptp_alloc_pins(pf);
15569         pci_set_drvdata(pdev, pf);
15570         pci_save_state(pdev);
15571
15572 #ifdef CONFIG_I40E_DCB
15573         status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15574         (!status &&
15575          lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15576                 (pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
15577                 (pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
15578         dev_info(&pdev->dev,
15579                  (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15580                         "FW LLDP is disabled\n" :
15581                         "FW LLDP is enabled\n");
15582
15583         /* Enable FW to write default DCB config on link-up */
15584         i40e_aq_set_dcb_parameters(hw, true, NULL);
15585
15586         err = i40e_init_pf_dcb(pf);
15587         if (err) {
15588                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15589                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15590                 /* Continue without DCB enabled */
15591         }
15592 #endif /* CONFIG_I40E_DCB */
15593
15594         /* set up periodic task facility */
15595         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15596         pf->service_timer_period = HZ;
15597
15598         INIT_WORK(&pf->service_task, i40e_service_task);
15599         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15600
15601         /* NVM bit on means WoL disabled for the port */
15602         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15603         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15604                 pf->wol_en = false;
15605         else
15606                 pf->wol_en = true;
15607         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15608
15609         /* set up the main switch operations */
15610         i40e_determine_queue_usage(pf);
15611         err = i40e_init_interrupt_scheme(pf);
15612         if (err)
15613                 goto err_switch_setup;
15614
15615         /* Reduce Tx and Rx pairs for kdump
15616          * When MSI-X is enabled, it's not allowed to use more TC queue
15617          * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15618          * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15619          */
15620         if (is_kdump_kernel())
15621                 pf->num_lan_msix = 1;
15622
15623         pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15624         pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15625         pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15626         pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15627         pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15628         pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15629                                                     UDP_TUNNEL_TYPE_GENEVE;
15630
15631         /* The number of VSIs reported by the FW is the minimum guaranteed
15632          * to us; HW supports far more and we share the remaining pool with
15633          * the other PFs. We allocate space for more than the guarantee with
15634          * the understanding that we might not get them all later.
15635          */
15636         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15637                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15638         else
15639                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15640         if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15641                 dev_warn(&pf->pdev->dev,
15642                          "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15643                          pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15644                 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15645         }
15646
15647         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15648         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15649                           GFP_KERNEL);
15650         if (!pf->vsi) {
15651                 err = -ENOMEM;
15652                 goto err_switch_setup;
15653         }
15654
15655 #ifdef CONFIG_PCI_IOV
15656         /* prep for VF support */
15657         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15658             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15659             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15660                 if (pci_num_vf(pdev))
15661                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15662         }
15663 #endif
15664         err = i40e_setup_pf_switch(pf, false, false);
15665         if (err) {
15666                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15667                 goto err_vsis;
15668         }
15669         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15670
15671         /* if FDIR VSI was set up, start it now */
15672         for (i = 0; i < pf->num_alloc_vsi; i++) {
15673                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15674                         i40e_vsi_open(pf->vsi[i]);
15675                         break;
15676                 }
15677         }
15678
15679         /* The driver only wants link up/down and module qualification
15680          * reports from firmware.  Note the negative logic.
15681          */
15682         err = i40e_aq_set_phy_int_mask(&pf->hw,
15683                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15684                                          I40E_AQ_EVENT_MEDIA_NA |
15685                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15686         if (err)
15687                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15688                          i40e_stat_str(&pf->hw, err),
15689                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15690
15691         /* Reconfigure hardware for allowing smaller MSS in the case
15692          * of TSO, so that we avoid the MDD being fired and causing
15693          * a reset in the case of small MSS+TSO.
15694          */
15695         val = rd32(hw, I40E_REG_MSS);
15696         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15697                 val &= ~I40E_REG_MSS_MIN_MASK;
15698                 val |= I40E_64BYTE_MSS;
15699                 wr32(hw, I40E_REG_MSS, val);
15700         }
15701
15702         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15703                 msleep(75);
15704                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15705                 if (err)
15706                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15707                                  i40e_stat_str(&pf->hw, err),
15708                                  i40e_aq_str(&pf->hw,
15709                                              pf->hw.aq.asq_last_status));
15710         }
15711         /* The main driver is (mostly) up and happy. We need to set this state
15712          * before setting up the misc vector or we get a race and the vector
15713          * ends up disabled forever.
15714          */
15715         clear_bit(__I40E_DOWN, pf->state);
15716
15717         /* In case of MSIX we are going to setup the misc vector right here
15718          * to handle admin queue events etc. In case of legacy and MSI
15719          * the misc functionality and queue processing is combined in
15720          * the same vector and that gets setup at open.
15721          */
15722         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15723                 err = i40e_setup_misc_vector(pf);
15724                 if (err) {
15725                         dev_info(&pdev->dev,
15726                                  "setup of misc vector failed: %d\n", err);
15727                         i40e_cloud_filter_exit(pf);
15728                         i40e_fdir_teardown(pf);
15729                         goto err_vsis;
15730                 }
15731         }
15732
15733 #ifdef CONFIG_PCI_IOV
15734         /* prep for VF support */
15735         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15736             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15737             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15738                 /* disable link interrupts for VFs */
15739                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15740                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15741                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15742                 i40e_flush(hw);
15743
15744                 if (pci_num_vf(pdev)) {
15745                         dev_info(&pdev->dev,
15746                                  "Active VFs found, allocating resources.\n");
15747                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15748                         if (err)
15749                                 dev_info(&pdev->dev,
15750                                          "Error %d allocating resources for existing VFs\n",
15751                                          err);
15752                 }
15753         }
15754 #endif /* CONFIG_PCI_IOV */
15755
15756         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15757                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15758                                                       pf->num_iwarp_msix,
15759                                                       I40E_IWARP_IRQ_PILE_ID);
15760                 if (pf->iwarp_base_vector < 0) {
15761                         dev_info(&pdev->dev,
15762                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
15763                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
15764                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15765                 }
15766         }
15767
15768         i40e_dbg_pf_init(pf);
15769
15770         /* tell the firmware that we're starting */
15771         i40e_send_version(pf);
15772
15773         /* since everything's happy, start the service_task timer */
15774         mod_timer(&pf->service_timer,
15775                   round_jiffies(jiffies + pf->service_timer_period));
15776
15777         /* add this PF to client device list and launch a client service task */
15778         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15779                 err = i40e_lan_add_device(pf);
15780                 if (err)
15781                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15782                                  err);
15783         }
15784
15785 #define PCI_SPEED_SIZE 8
15786 #define PCI_WIDTH_SIZE 8
15787         /* Devices on the IOSF bus do not have this information
15788          * and will report PCI Gen 1 x 1 by default so don't bother
15789          * checking them.
15790          */
15791         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15792                 char speed[PCI_SPEED_SIZE] = "Unknown";
15793                 char width[PCI_WIDTH_SIZE] = "Unknown";
15794
15795                 /* Get the negotiated link width and speed from PCI config
15796                  * space
15797                  */
15798                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15799                                           &link_status);
15800
15801                 i40e_set_pci_config_data(hw, link_status);
15802
15803                 switch (hw->bus.speed) {
15804                 case i40e_bus_speed_8000:
15805                         strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15806                 case i40e_bus_speed_5000:
15807                         strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15808                 case i40e_bus_speed_2500:
15809                         strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15810                 default:
15811                         break;
15812                 }
15813                 switch (hw->bus.width) {
15814                 case i40e_bus_width_pcie_x8:
15815                         strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15816                 case i40e_bus_width_pcie_x4:
15817                         strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15818                 case i40e_bus_width_pcie_x2:
15819                         strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15820                 case i40e_bus_width_pcie_x1:
15821                         strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15822                 default:
15823                         break;
15824                 }
15825
15826                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15827                          speed, width);
15828
15829                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15830                     hw->bus.speed < i40e_bus_speed_8000) {
15831                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15832                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15833                 }
15834         }
15835
15836         /* get the requested speeds from the fw */
15837         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15838         if (err)
15839                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
15840                         i40e_stat_str(&pf->hw, err),
15841                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15842         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15843
15844         /* set the FEC config due to the board capabilities */
15845         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15846
15847         /* get the supported phy types from the fw */
15848         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15849         if (err)
15850                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
15851                         i40e_stat_str(&pf->hw, err),
15852                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15853
15854         /* make sure the MFS hasn't been set lower than the default */
15855 #define MAX_FRAME_SIZE_DEFAULT 0x2600
15856         val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
15857                I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
15858         if (val < MAX_FRAME_SIZE_DEFAULT)
15859                 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
15860                          i, val);
15861
15862         /* Add a filter to drop all Flow control frames from any VSI from being
15863          * transmitted. By doing so we stop a malicious VF from sending out
15864          * PAUSE or PFC frames and potentially controlling traffic for other
15865          * PF/VF VSIs.
15866          * The FW can still send Flow control frames if enabled.
15867          */
15868         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15869                                                        pf->main_vsi_seid);
15870
15871         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15872                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15873                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15874         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15875                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15876         /* print a string summarizing features */
15877         i40e_print_features(pf);
15878
15879         return 0;
15880
15881         /* Unwind what we've done if something failed in the setup */
15882 err_vsis:
15883         set_bit(__I40E_DOWN, pf->state);
15884         i40e_clear_interrupt_scheme(pf);
15885         kfree(pf->vsi);
15886 err_switch_setup:
15887         i40e_reset_interrupt_capability(pf);
15888         del_timer_sync(&pf->service_timer);
15889 err_mac_addr:
15890 err_configure_lan_hmc:
15891         (void)i40e_shutdown_lan_hmc(hw);
15892 err_init_lan_hmc:
15893         kfree(pf->qp_pile);
15894 err_sw_init:
15895 err_adminq_setup:
15896 err_pf_reset:
15897         iounmap(hw->hw_addr);
15898 err_ioremap:
15899         kfree(pf);
15900 err_pf_alloc:
15901         pci_disable_pcie_error_reporting(pdev);
15902         pci_release_mem_regions(pdev);
15903 err_pci_reg:
15904 err_dma:
15905         pci_disable_device(pdev);
15906         return err;
15907 }
15908
15909 /**
15910  * i40e_remove - Device removal routine
15911  * @pdev: PCI device information struct
15912  *
15913  * i40e_remove is called by the PCI subsystem to alert the driver
15914  * that is should release a PCI device.  This could be caused by a
15915  * Hot-Plug event, or because the driver is going to be removed from
15916  * memory.
15917  **/
15918 static void i40e_remove(struct pci_dev *pdev)
15919 {
15920         struct i40e_pf *pf = pci_get_drvdata(pdev);
15921         struct i40e_hw *hw = &pf->hw;
15922         i40e_status ret_code;
15923         int i;
15924
15925         i40e_dbg_pf_exit(pf);
15926
15927         i40e_ptp_stop(pf);
15928
15929         /* Disable RSS in hw */
15930         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15931         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15932
15933         while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
15934                 usleep_range(1000, 2000);
15935
15936         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15937                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
15938                 i40e_free_vfs(pf);
15939                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15940         }
15941         /* no more scheduling of any task */
15942         set_bit(__I40E_SUSPENDED, pf->state);
15943         set_bit(__I40E_DOWN, pf->state);
15944         if (pf->service_timer.function)
15945                 del_timer_sync(&pf->service_timer);
15946         if (pf->service_task.func)
15947                 cancel_work_sync(&pf->service_task);
15948
15949         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15950                 struct i40e_vsi *vsi = pf->vsi[0];
15951
15952                 /* We know that we have allocated only one vsi for this PF,
15953                  * it was just for registering netdevice, so the interface
15954                  * could be visible in the 'ifconfig' output
15955                  */
15956                 unregister_netdev(vsi->netdev);
15957                 free_netdev(vsi->netdev);
15958
15959                 goto unmap;
15960         }
15961
15962         /* Client close must be called explicitly here because the timer
15963          * has been stopped.
15964          */
15965         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15966
15967         i40e_fdir_teardown(pf);
15968
15969         /* If there is a switch structure or any orphans, remove them.
15970          * This will leave only the PF's VSI remaining.
15971          */
15972         for (i = 0; i < I40E_MAX_VEB; i++) {
15973                 if (!pf->veb[i])
15974                         continue;
15975
15976                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15977                     pf->veb[i]->uplink_seid == 0)
15978                         i40e_switch_branch_release(pf->veb[i]);
15979         }
15980
15981         /* Now we can shutdown the PF's VSI, just before we kill
15982          * adminq and hmc.
15983          */
15984         if (pf->vsi[pf->lan_vsi])
15985                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
15986
15987         i40e_cloud_filter_exit(pf);
15988
15989         /* remove attached clients */
15990         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15991                 ret_code = i40e_lan_del_device(pf);
15992                 if (ret_code)
15993                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15994                                  ret_code);
15995         }
15996
15997         /* shutdown and destroy the HMC */
15998         if (hw->hmc.hmc_obj) {
15999                 ret_code = i40e_shutdown_lan_hmc(hw);
16000                 if (ret_code)
16001                         dev_warn(&pdev->dev,
16002                                  "Failed to destroy the HMC resources: %d\n",
16003                                  ret_code);
16004         }
16005
16006 unmap:
16007         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16008         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16009             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16010                 free_irq(pf->pdev->irq, pf);
16011
16012         /* shutdown the adminq */
16013         i40e_shutdown_adminq(hw);
16014
16015         /* destroy the locks only once, here */
16016         mutex_destroy(&hw->aq.arq_mutex);
16017         mutex_destroy(&hw->aq.asq_mutex);
16018
16019         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16020         rtnl_lock();
16021         i40e_clear_interrupt_scheme(pf);
16022         for (i = 0; i < pf->num_alloc_vsi; i++) {
16023                 if (pf->vsi[i]) {
16024                         if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16025                                 i40e_vsi_clear_rings(pf->vsi[i]);
16026                         i40e_vsi_clear(pf->vsi[i]);
16027                         pf->vsi[i] = NULL;
16028                 }
16029         }
16030         rtnl_unlock();
16031
16032         for (i = 0; i < I40E_MAX_VEB; i++) {
16033                 kfree(pf->veb[i]);
16034                 pf->veb[i] = NULL;
16035         }
16036
16037         kfree(pf->qp_pile);
16038         kfree(pf->vsi);
16039
16040         iounmap(hw->hw_addr);
16041         kfree(pf);
16042         pci_release_mem_regions(pdev);
16043
16044         pci_disable_pcie_error_reporting(pdev);
16045         pci_disable_device(pdev);
16046 }
16047
16048 /**
16049  * i40e_pci_error_detected - warning that something funky happened in PCI land
16050  * @pdev: PCI device information struct
16051  * @error: the type of PCI error
16052  *
16053  * Called to warn that something happened and the error handling steps
16054  * are in progress.  Allows the driver to quiesce things, be ready for
16055  * remediation.
16056  **/
16057 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16058                                                 pci_channel_state_t error)
16059 {
16060         struct i40e_pf *pf = pci_get_drvdata(pdev);
16061
16062         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16063
16064         if (!pf) {
16065                 dev_info(&pdev->dev,
16066                          "Cannot recover - error happened during device probe\n");
16067                 return PCI_ERS_RESULT_DISCONNECT;
16068         }
16069
16070         /* shutdown all operations */
16071         if (!test_bit(__I40E_SUSPENDED, pf->state))
16072                 i40e_prep_for_reset(pf);
16073
16074         /* Request a slot reset */
16075         return PCI_ERS_RESULT_NEED_RESET;
16076 }
16077
16078 /**
16079  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16080  * @pdev: PCI device information struct
16081  *
16082  * Called to find if the driver can work with the device now that
16083  * the pci slot has been reset.  If a basic connection seems good
16084  * (registers are readable and have sane content) then return a
16085  * happy little PCI_ERS_RESULT_xxx.
16086  **/
16087 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16088 {
16089         struct i40e_pf *pf = pci_get_drvdata(pdev);
16090         pci_ers_result_t result;
16091         u32 reg;
16092
16093         dev_dbg(&pdev->dev, "%s\n", __func__);
16094         if (pci_enable_device_mem(pdev)) {
16095                 dev_info(&pdev->dev,
16096                          "Cannot re-enable PCI device after reset.\n");
16097                 result = PCI_ERS_RESULT_DISCONNECT;
16098         } else {
16099                 pci_set_master(pdev);
16100                 pci_restore_state(pdev);
16101                 pci_save_state(pdev);
16102                 pci_wake_from_d3(pdev, false);
16103
16104                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16105                 if (reg == 0)
16106                         result = PCI_ERS_RESULT_RECOVERED;
16107                 else
16108                         result = PCI_ERS_RESULT_DISCONNECT;
16109         }
16110
16111         return result;
16112 }
16113
16114 /**
16115  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16116  * @pdev: PCI device information struct
16117  */
16118 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16119 {
16120         struct i40e_pf *pf = pci_get_drvdata(pdev);
16121
16122         i40e_prep_for_reset(pf);
16123 }
16124
16125 /**
16126  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16127  * @pdev: PCI device information struct
16128  */
16129 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16130 {
16131         struct i40e_pf *pf = pci_get_drvdata(pdev);
16132
16133         i40e_reset_and_rebuild(pf, false, false);
16134 }
16135
16136 /**
16137  * i40e_pci_error_resume - restart operations after PCI error recovery
16138  * @pdev: PCI device information struct
16139  *
16140  * Called to allow the driver to bring things back up after PCI error
16141  * and/or reset recovery has finished.
16142  **/
16143 static void i40e_pci_error_resume(struct pci_dev *pdev)
16144 {
16145         struct i40e_pf *pf = pci_get_drvdata(pdev);
16146
16147         dev_dbg(&pdev->dev, "%s\n", __func__);
16148         if (test_bit(__I40E_SUSPENDED, pf->state))
16149                 return;
16150
16151         i40e_handle_reset_warning(pf, false);
16152 }
16153
16154 /**
16155  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16156  * using the mac_address_write admin q function
16157  * @pf: pointer to i40e_pf struct
16158  **/
16159 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16160 {
16161         struct i40e_hw *hw = &pf->hw;
16162         i40e_status ret;
16163         u8 mac_addr[6];
16164         u16 flags = 0;
16165
16166         /* Get current MAC address in case it's an LAA */
16167         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16168                 ether_addr_copy(mac_addr,
16169                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16170         } else {
16171                 dev_err(&pf->pdev->dev,
16172                         "Failed to retrieve MAC address; using default\n");
16173                 ether_addr_copy(mac_addr, hw->mac.addr);
16174         }
16175
16176         /* The FW expects the mac address write cmd to first be called with
16177          * one of these flags before calling it again with the multicast
16178          * enable flags.
16179          */
16180         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16181
16182         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16183                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16184
16185         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16186         if (ret) {
16187                 dev_err(&pf->pdev->dev,
16188                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16189                 return;
16190         }
16191
16192         flags = I40E_AQC_MC_MAG_EN
16193                         | I40E_AQC_WOL_PRESERVE_ON_PFR
16194                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16195         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16196         if (ret)
16197                 dev_err(&pf->pdev->dev,
16198                         "Failed to enable Multicast Magic Packet wake up\n");
16199 }
16200
16201 /**
16202  * i40e_shutdown - PCI callback for shutting down
16203  * @pdev: PCI device information struct
16204  **/
16205 static void i40e_shutdown(struct pci_dev *pdev)
16206 {
16207         struct i40e_pf *pf = pci_get_drvdata(pdev);
16208         struct i40e_hw *hw = &pf->hw;
16209
16210         set_bit(__I40E_SUSPENDED, pf->state);
16211         set_bit(__I40E_DOWN, pf->state);
16212
16213         del_timer_sync(&pf->service_timer);
16214         cancel_work_sync(&pf->service_task);
16215         i40e_cloud_filter_exit(pf);
16216         i40e_fdir_teardown(pf);
16217
16218         /* Client close must be called explicitly here because the timer
16219          * has been stopped.
16220          */
16221         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16222
16223         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16224                 i40e_enable_mc_magic_wake(pf);
16225
16226         i40e_prep_for_reset(pf);
16227
16228         wr32(hw, I40E_PFPM_APM,
16229              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16230         wr32(hw, I40E_PFPM_WUFC,
16231              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16232
16233         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16234         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16235             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16236                 free_irq(pf->pdev->irq, pf);
16237
16238         /* Since we're going to destroy queues during the
16239          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16240          * whole section
16241          */
16242         rtnl_lock();
16243         i40e_clear_interrupt_scheme(pf);
16244         rtnl_unlock();
16245
16246         if (system_state == SYSTEM_POWER_OFF) {
16247                 pci_wake_from_d3(pdev, pf->wol_en);
16248                 pci_set_power_state(pdev, PCI_D3hot);
16249         }
16250 }
16251
16252 /**
16253  * i40e_suspend - PM callback for moving to D3
16254  * @dev: generic device information structure
16255  **/
16256 static int __maybe_unused i40e_suspend(struct device *dev)
16257 {
16258         struct i40e_pf *pf = dev_get_drvdata(dev);
16259         struct i40e_hw *hw = &pf->hw;
16260
16261         /* If we're already suspended, then there is nothing to do */
16262         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16263                 return 0;
16264
16265         set_bit(__I40E_DOWN, pf->state);
16266
16267         /* Ensure service task will not be running */
16268         del_timer_sync(&pf->service_timer);
16269         cancel_work_sync(&pf->service_task);
16270
16271         /* Client close must be called explicitly here because the timer
16272          * has been stopped.
16273          */
16274         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16275
16276         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16277                 i40e_enable_mc_magic_wake(pf);
16278
16279         /* Since we're going to destroy queues during the
16280          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16281          * whole section
16282          */
16283         rtnl_lock();
16284
16285         i40e_prep_for_reset(pf);
16286
16287         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16288         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16289
16290         /* Clear the interrupt scheme and release our IRQs so that the system
16291          * can safely hibernate even when there are a large number of CPUs.
16292          * Otherwise hibernation might fail when mapping all the vectors back
16293          * to CPU0.
16294          */
16295         i40e_clear_interrupt_scheme(pf);
16296
16297         rtnl_unlock();
16298
16299         return 0;
16300 }
16301
16302 /**
16303  * i40e_resume - PM callback for waking up from D3
16304  * @dev: generic device information structure
16305  **/
16306 static int __maybe_unused i40e_resume(struct device *dev)
16307 {
16308         struct i40e_pf *pf = dev_get_drvdata(dev);
16309         int err;
16310
16311         /* If we're not suspended, then there is nothing to do */
16312         if (!test_bit(__I40E_SUSPENDED, pf->state))
16313                 return 0;
16314
16315         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16316          * since we're going to be restoring queues
16317          */
16318         rtnl_lock();
16319
16320         /* We cleared the interrupt scheme when we suspended, so we need to
16321          * restore it now to resume device functionality.
16322          */
16323         err = i40e_restore_interrupt_scheme(pf);
16324         if (err) {
16325                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16326                         err);
16327         }
16328
16329         clear_bit(__I40E_DOWN, pf->state);
16330         i40e_reset_and_rebuild(pf, false, true);
16331
16332         rtnl_unlock();
16333
16334         /* Clear suspended state last after everything is recovered */
16335         clear_bit(__I40E_SUSPENDED, pf->state);
16336
16337         /* Restart the service task */
16338         mod_timer(&pf->service_timer,
16339                   round_jiffies(jiffies + pf->service_timer_period));
16340
16341         return 0;
16342 }
16343
16344 static const struct pci_error_handlers i40e_err_handler = {
16345         .error_detected = i40e_pci_error_detected,
16346         .slot_reset = i40e_pci_error_slot_reset,
16347         .reset_prepare = i40e_pci_error_reset_prepare,
16348         .reset_done = i40e_pci_error_reset_done,
16349         .resume = i40e_pci_error_resume,
16350 };
16351
16352 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16353
16354 static struct pci_driver i40e_driver = {
16355         .name     = i40e_driver_name,
16356         .id_table = i40e_pci_tbl,
16357         .probe    = i40e_probe,
16358         .remove   = i40e_remove,
16359         .driver   = {
16360                 .pm = &i40e_pm_ops,
16361         },
16362         .shutdown = i40e_shutdown,
16363         .err_handler = &i40e_err_handler,
16364         .sriov_configure = i40e_pci_sriov_configure,
16365 };
16366
16367 /**
16368  * i40e_init_module - Driver registration routine
16369  *
16370  * i40e_init_module is the first routine called when the driver is
16371  * loaded. All it does is register with the PCI subsystem.
16372  **/
16373 static int __init i40e_init_module(void)
16374 {
16375         pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16376         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16377
16378         /* There is no need to throttle the number of active tasks because
16379          * each device limits its own task using a state bit for scheduling
16380          * the service task, and the device tasks do not interfere with each
16381          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16382          * since we need to be able to guarantee forward progress even under
16383          * memory pressure.
16384          */
16385         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16386         if (!i40e_wq) {
16387                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16388                 return -ENOMEM;
16389         }
16390
16391         i40e_dbg_init();
16392         return pci_register_driver(&i40e_driver);
16393 }
16394 module_init(i40e_init_module);
16395
16396 /**
16397  * i40e_exit_module - Driver exit cleanup routine
16398  *
16399  * i40e_exit_module is called just before the driver is removed
16400  * from memory.
16401  **/
16402 static void __exit i40e_exit_module(void)
16403 {
16404         pci_unregister_driver(&i40e_driver);
16405         destroy_workqueue(i40e_wq);
16406         ida_destroy(&i40e_client_ida);
16407         i40e_dbg_exit();
16408 }
16409 module_exit(i40e_exit_module);