drivers/net/ethernet/intel/iavf/iavf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include "iavf.h"
   5 #include "iavf_prototype.h"
   6 #include "iavf_client.h"
   7 /* All iavf tracepoints are defined by the include below, which must
   8  * be included exactly once across the whole kernel with
   9  * CREATE_TRACE_POINTS defined
  10  */
  11 #define CREATE_TRACE_POINTS
  12 #include "iavf_trace.h"
  13
  14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
  15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
  16 static int iavf_close(struct net_device *netdev);
  17 static void iavf_init_get_resources(struct iavf_adapter *adapter);
  18 static int iavf_check_reset_complete(struct iavf_hw *hw);
  19
  20 char iavf_driver_name[] = "iavf";
  21 static const char iavf_driver_string[] =
  22         "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
  23
  24 static const char iavf_copyright[] =
  25         "Copyright (c) 2013 - 2018 Intel Corporation.";
  26
  27 /* iavf_pci_tbl - PCI Device ID Table
  28  *
  29  * Wildcard entries (PCI_ANY_ID) should come last
  30  * Last entry must be all 0s
  31  *
  32  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  33  *   Class, Class Mask, private data (not used) }
  34  */
  35 static const struct pci_device_id iavf_pci_tbl[] = {
  36         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
  37         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
  38         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
  39         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
  40         /* required last entry */
  41         {0, }
  42 };
  43
  44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
  45
  46 MODULE_ALIAS("i40evf");
  47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
  49 MODULE_LICENSE("GPL v2");
  50
  51 static const struct net_device_ops iavf_netdev_ops;
  52 struct workqueue_struct *iavf_wq;
  53
  54 int iavf_status_to_errno(enum iavf_status status)
  55 {
  56         switch (status) {
  57         case IAVF_SUCCESS:
  58                 return 0;
  59         case IAVF_ERR_PARAM:
  60         case IAVF_ERR_MAC_TYPE:
  61         case IAVF_ERR_INVALID_MAC_ADDR:
  62         case IAVF_ERR_INVALID_LINK_SETTINGS:
  63         case IAVF_ERR_INVALID_PD_ID:
  64         case IAVF_ERR_INVALID_QP_ID:
  65         case IAVF_ERR_INVALID_CQ_ID:
  66         case IAVF_ERR_INVALID_CEQ_ID:
  67         case IAVF_ERR_INVALID_AEQ_ID:
  68         case IAVF_ERR_INVALID_SIZE:
  69         case IAVF_ERR_INVALID_ARP_INDEX:
  70         case IAVF_ERR_INVALID_FPM_FUNC_ID:
  71         case IAVF_ERR_QP_INVALID_MSG_SIZE:
  72         case IAVF_ERR_INVALID_FRAG_COUNT:
  73         case IAVF_ERR_INVALID_ALIGNMENT:
  74         case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
  75         case IAVF_ERR_INVALID_IMM_DATA_SIZE:
  76         case IAVF_ERR_INVALID_VF_ID:
  77         case IAVF_ERR_INVALID_HMCFN_ID:
  78         case IAVF_ERR_INVALID_PBLE_INDEX:
  79         case IAVF_ERR_INVALID_SD_INDEX:
  80         case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
  81         case IAVF_ERR_INVALID_SD_TYPE:
  82         case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
  83         case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
  84         case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
  85                 return -EINVAL;
  86         case IAVF_ERR_NVM:
  87         case IAVF_ERR_NVM_CHECKSUM:
  88         case IAVF_ERR_PHY:
  89         case IAVF_ERR_CONFIG:
  90         case IAVF_ERR_UNKNOWN_PHY:
  91         case IAVF_ERR_LINK_SETUP:
  92         case IAVF_ERR_ADAPTER_STOPPED:
  93         case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
  94         case IAVF_ERR_AUTONEG_NOT_COMPLETE:
  95         case IAVF_ERR_RESET_FAILED:
  96         case IAVF_ERR_BAD_PTR:
  97         case IAVF_ERR_SWFW_SYNC:
  98         case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
  99         case IAVF_ERR_QUEUE_EMPTY:
 100         case IAVF_ERR_FLUSHED_QUEUE:
 101         case IAVF_ERR_OPCODE_MISMATCH:
 102         case IAVF_ERR_CQP_COMPL_ERROR:
 103         case IAVF_ERR_BACKING_PAGE_ERROR:
 104         case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
 105         case IAVF_ERR_MEMCPY_FAILED:
 106         case IAVF_ERR_SRQ_ENABLED:
 107         case IAVF_ERR_ADMIN_QUEUE_ERROR:
 108         case IAVF_ERR_ADMIN_QUEUE_FULL:
 109         case IAVF_ERR_BAD_IWARP_CQE:
 110         case IAVF_ERR_NVM_BLANK_MODE:
 111         case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
 112         case IAVF_ERR_DIAG_TEST_FAILED:
 113         case IAVF_ERR_FIRMWARE_API_VERSION:
 114         case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
 115                 return -EIO;
 116         case IAVF_ERR_DEVICE_NOT_SUPPORTED:
 117                 return -ENODEV;
 118         case IAVF_ERR_NO_AVAILABLE_VSI:
 119         case IAVF_ERR_RING_FULL:
 120                 return -ENOSPC;
 121         case IAVF_ERR_NO_MEMORY:
 122                 return -ENOMEM;
 123         case IAVF_ERR_TIMEOUT:
 124         case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
 125                 return -ETIMEDOUT;
 126         case IAVF_ERR_NOT_IMPLEMENTED:
 127         case IAVF_NOT_SUPPORTED:
 128                 return -EOPNOTSUPP;
 129         case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
 130                 return -EALREADY;
 131         case IAVF_ERR_NOT_READY:
 132                 return -EBUSY;
 133         case IAVF_ERR_BUF_TOO_SHORT:
 134                 return -EMSGSIZE;
 135         }
 136
 137         return -EIO;
 138 }
 139
 140 int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
 141 {
 142         switch (v_status) {
 143         case VIRTCHNL_STATUS_SUCCESS:
 144                 return 0;
 145         case VIRTCHNL_STATUS_ERR_PARAM:
 146         case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
 147                 return -EINVAL;
 148         case VIRTCHNL_STATUS_ERR_NO_MEMORY:
 149                 return -ENOMEM;
 150         case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
 151         case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
 152         case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
 153                 return -EIO;
 154         case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
 155                 return -EOPNOTSUPP;
 156         }
 157
 158         return -EIO;
 159 }
 160
 161 /**
 162  * iavf_pdev_to_adapter - go from pci_dev to adapter
 163  * @pdev: pci_dev pointer
 164  */
 165 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
 166 {
 167         return netdev_priv(pci_get_drvdata(pdev));
 168 }
 169
 170 /**
 171  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
 172  * @hw:   pointer to the HW structure
 173  * @mem:  ptr to mem struct to fill out
 174  * @size: size of memory requested
 175  * @alignment: what to align the allocation to
 176  **/
 177 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
 178                                          struct iavf_dma_mem *mem,
 179                                          u64 size, u32 alignment)
 180 {
 181         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
 182
 183         if (!mem)
 184                 return IAVF_ERR_PARAM;
 185
 186         mem->size = ALIGN(size, alignment);
 187         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
 188                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
 189         if (mem->va)
 190                 return 0;
 191         else
 192                 return IAVF_ERR_NO_MEMORY;
 193 }
 194
 195 /**
 196  * iavf_free_dma_mem_d - OS specific memory free for shared code
 197  * @hw:   pointer to the HW structure
 198  * @mem:  ptr to mem struct to free
 199  **/
 200 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw,
 201                                      struct iavf_dma_mem *mem)
 202 {
 203         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
 204
 205         if (!mem || !mem->va)
 206                 return IAVF_ERR_PARAM;
 207         dma_free_coherent(&adapter->pdev->dev, mem->size,
 208                           mem->va, (dma_addr_t)mem->pa);
 209         return 0;
 210 }
 211
 212 /**
 213  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
 214  * @hw:   pointer to the HW structure
 215  * @mem:  ptr to mem struct to fill out
 216  * @size: size of memory requested
 217  **/
 218 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw,
 219                                           struct iavf_virt_mem *mem, u32 size)
 220 {
 221         if (!mem)
 222                 return IAVF_ERR_PARAM;
 223
 224         mem->size = size;
 225         mem->va = kzalloc(size, GFP_KERNEL);
 226
 227         if (mem->va)
 228                 return 0;
 229         else
 230                 return IAVF_ERR_NO_MEMORY;
 231 }
 232
 233 /**
 234  * iavf_free_virt_mem_d - OS specific memory free for shared code
 235  * @hw:   pointer to the HW structure
 236  * @mem:  ptr to mem struct to free
 237  **/
 238 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
 239                                       struct iavf_virt_mem *mem)
 240 {
 241         if (!mem)
 242                 return IAVF_ERR_PARAM;
 243
 244         /* it's ok to kfree a NULL pointer */
 245         kfree(mem->va);
 246
 247         return 0;
 248 }
 249
 250 /**
 251  * iavf_lock_timeout - try to lock mutex but give up after timeout
 252  * @lock: mutex that should be locked
 253  * @msecs: timeout in msecs
 254  *
 255  * Returns 0 on success, negative on failure
 256  **/
 257 int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
 258 {
 259         unsigned int wait, delay = 10;
 260
 261         for (wait = 0; wait < msecs; wait += delay) {
 262                 if (mutex_trylock(lock))
 263                         return 0;
 264
 265                 msleep(delay);
 266         }
 267
 268         return -1;
 269 }
 270
 271 /**
 272  * iavf_schedule_reset - Set the flags and schedule a reset event
 273  * @adapter: board private structure
 274  **/
 275 void iavf_schedule_reset(struct iavf_adapter *adapter)
 276 {
 277         if (!(adapter->flags &
 278               (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
 279                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
 280                 queue_work(iavf_wq, &adapter->reset_task);
 281         }
 282 }
 283
 284 /**
 285  * iavf_schedule_request_stats - Set the flags and schedule statistics request
 286  * @adapter: board private structure
 287  *
 288  * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly
 289  * request and refresh ethtool stats
 290  **/
 291 void iavf_schedule_request_stats(struct iavf_adapter *adapter)
 292 {
 293         adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS;
 294         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
 295 }
 296
 297 /**
 298  * iavf_tx_timeout - Respond to a Tx Hang
 299  * @netdev: network interface device structure
 300  * @txqueue: queue number that is timing out
 301  **/
 302 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 303 {
 304         struct iavf_adapter *adapter = netdev_priv(netdev);
 305
 306         adapter->tx_timeout_count++;
 307         iavf_schedule_reset(adapter);
 308 }
 309
 310 /**
 311  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
 312  * @adapter: board private structure
 313  **/
 314 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
 315 {
 316         struct iavf_hw *hw = &adapter->hw;
 317
 318         if (!adapter->msix_entries)
 319                 return;
 320
 321         wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
 322
 323         iavf_flush(hw);
 324
 325         synchronize_irq(adapter->msix_entries[0].vector);
 326 }
 327
 328 /**
 329  * iavf_misc_irq_enable - Enable default interrupt generation settings
 330  * @adapter: board private structure
 331  **/
 332 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
 333 {
 334         struct iavf_hw *hw = &adapter->hw;
 335
 336         wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
 337                                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
 338         wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
 339
 340         iavf_flush(hw);
 341 }
 342
 343 /**
 344  * iavf_irq_disable - Mask off interrupt generation on the NIC
 345  * @adapter: board private structure
 346  **/
 347 static void iavf_irq_disable(struct iavf_adapter *adapter)
 348 {
 349         int i;
 350         struct iavf_hw *hw = &adapter->hw;
 351
 352         if (!adapter->msix_entries)
 353                 return;
 354
 355         for (i = 1; i < adapter->num_msix_vectors; i++) {
 356                 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
 357                 synchronize_irq(adapter->msix_entries[i].vector);
 358         }
 359         iavf_flush(hw);
 360 }
 361
 362 /**
 363  * iavf_irq_enable_queues - Enable interrupt for specified queues
 364  * @adapter: board private structure
 365  * @mask: bitmap of queues to enable
 366  **/
 367 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
 368 {
 369         struct iavf_hw *hw = &adapter->hw;
 370         int i;
 371
 372         for (i = 1; i < adapter->num_msix_vectors; i++) {
 373                 if (mask & BIT(i - 1)) {
 374                         wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
 375                              IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
 376                              IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 377                 }
 378         }
 379 }
 380
 381 /**
 382  * iavf_irq_enable - Enable default interrupt generation settings
 383  * @adapter: board private structure
 384  * @flush: boolean value whether to run rd32()
 385  **/
 386 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
 387 {
 388         struct iavf_hw *hw = &adapter->hw;
 389
 390         iavf_misc_irq_enable(adapter);
 391         iavf_irq_enable_queues(adapter, ~0);
 392
 393         if (flush)
 394                 iavf_flush(hw);
 395 }
 396
 397 /**
 398  * iavf_msix_aq - Interrupt handler for vector 0
 399  * @irq: interrupt number
 400  * @data: pointer to netdev
 401  **/
 402 static irqreturn_t iavf_msix_aq(int irq, void *data)
 403 {
 404         struct net_device *netdev = data;
 405         struct iavf_adapter *adapter = netdev_priv(netdev);
 406         struct iavf_hw *hw = &adapter->hw;
 407
 408         /* handle non-queue interrupts, these reads clear the registers */
 409         rd32(hw, IAVF_VFINT_ICR01);
 410         rd32(hw, IAVF_VFINT_ICR0_ENA1);
 411
 412         if (adapter->state != __IAVF_REMOVE)
 413                 /* schedule work on the private workqueue */
 414                 queue_work(iavf_wq, &adapter->adminq_task);
 415
 416         return IRQ_HANDLED;
 417 }
 418
 419 /**
 420  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
 421  * @irq: interrupt number
 422  * @data: pointer to a q_vector
 423  **/
 424 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
 425 {
 426         struct iavf_q_vector *q_vector = data;
 427
 428         if (!q_vector->tx.ring && !q_vector->rx.ring)
 429                 return IRQ_HANDLED;
 430
 431         napi_schedule_irqoff(&q_vector->napi);
 432
 433         return IRQ_HANDLED;
 434 }
 435
 436 /**
 437  * iavf_map_vector_to_rxq - associate irqs with rx queues
 438  * @adapter: board private structure
 439  * @v_idx: interrupt number
 440  * @r_idx: queue number
 441  **/
 442 static void
 443 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
 444 {
 445         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 446         struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
 447         struct iavf_hw *hw = &adapter->hw;
 448
 449         rx_ring->q_vector = q_vector;
 450         rx_ring->next = q_vector->rx.ring;
 451         rx_ring->vsi = &adapter->vsi;
 452         q_vector->rx.ring = rx_ring;
 453         q_vector->rx.count++;
 454         q_vector->rx.next_update = jiffies + 1;
 455         q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
 456         q_vector->ring_mask |= BIT(r_idx);
 457         wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
 458              q_vector->rx.current_itr >> 1);
 459         q_vector->rx.current_itr = q_vector->rx.target_itr;
 460 }
 461
 462 /**
 463  * iavf_map_vector_to_txq - associate irqs with tx queues
 464  * @adapter: board private structure
 465  * @v_idx: interrupt number
 466  * @t_idx: queue number
 467  **/
 468 static void
 469 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
 470 {
 471         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 472         struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
 473         struct iavf_hw *hw = &adapter->hw;
 474
 475         tx_ring->q_vector = q_vector;
 476         tx_ring->next = q_vector->tx.ring;
 477         tx_ring->vsi = &adapter->vsi;
 478         q_vector->tx.ring = tx_ring;
 479         q_vector->tx.count++;
 480         q_vector->tx.next_update = jiffies + 1;
 481         q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
 482         q_vector->num_ringpairs++;
 483         wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
 484              q_vector->tx.target_itr >> 1);
 485         q_vector->tx.current_itr = q_vector->tx.target_itr;
 486 }
 487
 488 /**
 489  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
 490  * @adapter: board private structure to initialize
 491  *
 492  * This function maps descriptor rings to the queue-specific vectors
 493  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 494  * one vector per ring/queue, but on a constrained vector budget, we
 495  * group the rings as "efficiently" as possible.  You would add new
 496  * mapping configurations in here.
 497  **/
 498 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
 499 {
 500         int rings_remaining = adapter->num_active_queues;
 501         int ridx = 0, vidx = 0;
 502         int q_vectors;
 503
 504         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 505
 506         for (; ridx < rings_remaining; ridx++) {
 507                 iavf_map_vector_to_rxq(adapter, vidx, ridx);
 508                 iavf_map_vector_to_txq(adapter, vidx, ridx);
 509
 510                 /* In the case where we have more queues than vectors, continue
 511                  * round-robin on vectors until all queues are mapped.
 512                  */
 513                 if (++vidx >= q_vectors)
 514                         vidx = 0;
 515         }
 516
 517         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
 518 }
 519
 520 /**
 521  * iavf_irq_affinity_notify - Callback for affinity changes
 522  * @notify: context as to what irq was changed
 523  * @mask: the new affinity mask
 524  *
 525  * This is a callback function used by the irq_set_affinity_notifier function
 526  * so that we may register to receive changes to the irq affinity masks.
 527  **/
 528 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
 529                                      const cpumask_t *mask)
 530 {
 531         struct iavf_q_vector *q_vector =
 532                 container_of(notify, struct iavf_q_vector, affinity_notify);
 533
 534         cpumask_copy(&q_vector->affinity_mask, mask);
 535 }
 536
 537 /**
 538  * iavf_irq_affinity_release - Callback for affinity notifier release
 539  * @ref: internal core kernel usage
 540  *
 541  * This is a callback function used by the irq_set_affinity_notifier function
 542  * to inform the current notification subscriber that they will no longer
 543  * receive notifications.
 544  **/
 545 static void iavf_irq_affinity_release(struct kref *ref) {}
 546
 547 /**
 548  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
 549  * @adapter: board private structure
 550  * @basename: device basename
 551  *
 552  * Allocates MSI-X vectors for tx and rx handling, and requests
 553  * interrupts from the kernel.
 554  **/
 555 static int
 556 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
 557 {
 558         unsigned int vector, q_vectors;
 559         unsigned int rx_int_idx = 0, tx_int_idx = 0;
 560         int irq_num, err;
 561         int cpu;
 562
 563         iavf_irq_disable(adapter);
 564         /* Decrement for Other and TCP Timer vectors */
 565         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 566
 567         for (vector = 0; vector < q_vectors; vector++) {
 568                 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
 569
 570                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 571
 572                 if (q_vector->tx.ring && q_vector->rx.ring) {
 573                         snprintf(q_vector->name, sizeof(q_vector->name),
 574                                  "iavf-%s-TxRx-%u", basename, rx_int_idx++);
 575                         tx_int_idx++;
 576                 } else if (q_vector->rx.ring) {
 577                         snprintf(q_vector->name, sizeof(q_vector->name),
 578                                  "iavf-%s-rx-%u", basename, rx_int_idx++);
 579                 } else if (q_vector->tx.ring) {
 580                         snprintf(q_vector->name, sizeof(q_vector->name),
 581                                  "iavf-%s-tx-%u", basename, tx_int_idx++);
 582                 } else {
 583                         /* skip this unused q_vector */
 584                         continue;
 585                 }
 586                 err = request_irq(irq_num,
 587                                   iavf_msix_clean_rings,
 588                                   0,
 589                                   q_vector->name,
 590                                   q_vector);
 591                 if (err) {
 592                         dev_info(&adapter->pdev->dev,
 593                                  "Request_irq failed, error: %d\n", err);
 594                         goto free_queue_irqs;
 595                 }
 596                 /* register for affinity change notifications */
 597                 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
 598                 q_vector->affinity_notify.release =
 599                                                    iavf_irq_affinity_release;
 600                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
 601                 /* Spread the IRQ affinity hints across online CPUs. Note that
 602                  * get_cpu_mask returns a mask with a permanent lifetime so
 603                  * it's safe to use as a hint for irq_update_affinity_hint.
 604                  */
 605                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
 606                 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
 607         }
 608
 609         return 0;
 610
 611 free_queue_irqs:
 612         while (vector) {
 613                 vector--;
 614                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 615                 irq_set_affinity_notifier(irq_num, NULL);
 616                 irq_update_affinity_hint(irq_num, NULL);
 617                 free_irq(irq_num, &adapter->q_vectors[vector]);
 618         }
 619         return err;
 620 }
 621
 622 /**
 623  * iavf_request_misc_irq - Initialize MSI-X interrupts
 624  * @adapter: board private structure
 625  *
 626  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 627  * vector is only for the admin queue, and stays active even when the netdev
 628  * is closed.
 629  **/
 630 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
 631 {
 632         struct net_device *netdev = adapter->netdev;
 633         int err;
 634
 635         snprintf(adapter->misc_vector_name,
 636                  sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
 637                  dev_name(&adapter->pdev->dev));
 638         err = request_irq(adapter->msix_entries[0].vector,
 639                           &iavf_msix_aq, 0,
 640                           adapter->misc_vector_name, netdev);
 641         if (err) {
 642                 dev_err(&adapter->pdev->dev,
 643                         "request_irq for %s failed: %d\n",
 644                         adapter->misc_vector_name, err);
 645                 free_irq(adapter->msix_entries[0].vector, netdev);
 646         }
 647         return err;
 648 }
 649
 650 /**
 651  * iavf_free_traffic_irqs - Free MSI-X interrupts
 652  * @adapter: board private structure
 653  *
 654  * Frees all MSI-X vectors other than 0.
 655  **/
 656 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
 657 {
 658         int vector, irq_num, q_vectors;
 659
 660         if (!adapter->msix_entries)
 661                 return;
 662
 663         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 664
 665         for (vector = 0; vector < q_vectors; vector++) {
 666                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 667                 irq_set_affinity_notifier(irq_num, NULL);
 668                 irq_update_affinity_hint(irq_num, NULL);
 669                 free_irq(irq_num, &adapter->q_vectors[vector]);
 670         }
 671 }
 672
 673 /**
 674  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
 675  * @adapter: board private structure
 676  *
 677  * Frees MSI-X vector 0.
 678  **/
 679 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
 680 {
 681         struct net_device *netdev = adapter->netdev;
 682
 683         if (!adapter->msix_entries)
 684                 return;
 685
 686         free_irq(adapter->msix_entries[0].vector, netdev);
 687 }
 688
 689 /**
 690  * iavf_configure_tx - Configure Transmit Unit after Reset
 691  * @adapter: board private structure
 692  *
 693  * Configure the Tx unit of the MAC after a reset.
 694  **/
 695 static void iavf_configure_tx(struct iavf_adapter *adapter)
 696 {
 697         struct iavf_hw *hw = &adapter->hw;
 698         int i;
 699
 700         for (i = 0; i < adapter->num_active_queues; i++)
 701                 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
 702 }
 703
 704 /**
 705  * iavf_configure_rx - Configure Receive Unit after Reset
 706  * @adapter: board private structure
 707  *
 708  * Configure the Rx unit of the MAC after a reset.
 709  **/
 710 static void iavf_configure_rx(struct iavf_adapter *adapter)
 711 {
 712         unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
 713         struct iavf_hw *hw = &adapter->hw;
 714         int i;
 715
 716         /* Legacy Rx will always default to a 2048 buffer size. */
 717 #if (PAGE_SIZE < 8192)
 718         if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
 719                 struct net_device *netdev = adapter->netdev;
 720
 721                 /* For jumbo frames on systems with 4K pages we have to use
 722                  * an order 1 page, so we might as well increase the size
 723                  * of our Rx buffer to make better use of the available space
 724                  */
 725                 rx_buf_len = IAVF_RXBUFFER_3072;
 726
 727                 /* We use a 1536 buffer size for configurations with
 728                  * standard Ethernet mtu.  On x86 this gives us enough room
 729                  * for shared info and 192 bytes of padding.
 730                  */
 731                 if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
 732                     (netdev->mtu <= ETH_DATA_LEN))
 733                         rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
 734         }
 735 #endif
 736
 737         for (i = 0; i < adapter->num_active_queues; i++) {
 738                 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
 739                 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
 740
 741                 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
 742                         clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
 743                 else
 744                         set_ring_build_skb_enabled(&adapter->rx_rings[i]);
 745         }
 746 }
 747
 748 /**
 749  * iavf_find_vlan - Search filter list for specific vlan filter
 750  * @adapter: board private structure
 751  * @vlan: vlan tag
 752  *
 753  * Returns ptr to the filter object or NULL. Must be called while holding the
 754  * mac_vlan_list_lock.
 755  **/
 756 static struct
 757 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
 758                                  struct iavf_vlan vlan)
 759 {
 760         struct iavf_vlan_filter *f;
 761
 762         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 763                 if (f->vlan.vid == vlan.vid &&
 764                     f->vlan.tpid == vlan.tpid)
 765                         return f;
 766         }
 767
 768         return NULL;
 769 }
 770
 771 /**
 772  * iavf_add_vlan - Add a vlan filter to the list
 773  * @adapter: board private structure
 774  * @vlan: VLAN tag
 775  *
 776  * Returns ptr to the filter object or NULL when no memory available.
 777  **/
 778 static struct
 779 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
 780                                 struct iavf_vlan vlan)
 781 {
 782         struct iavf_vlan_filter *f = NULL;
 783
 784         spin_lock_bh(&adapter->mac_vlan_list_lock);
 785
 786         f = iavf_find_vlan(adapter, vlan);
 787         if (!f) {
 788                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 789                 if (!f)
 790                         goto clearout;
 791
 792                 f->vlan = vlan;
 793
 794                 list_add_tail(&f->list, &adapter->vlan_filter_list);
 795                 f->add = true;
 796                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
 797         }
 798
 799 clearout:
 800         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 801         return f;
 802 }
 803
 804 /**
 805  * iavf_del_vlan - Remove a vlan filter from the list
 806  * @adapter: board private structure
 807  * @vlan: VLAN tag
 808  **/
 809 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
 810 {
 811         struct iavf_vlan_filter *f;
 812
 813         spin_lock_bh(&adapter->mac_vlan_list_lock);
 814
 815         f = iavf_find_vlan(adapter, vlan);
 816         if (f) {
 817                 f->remove = true;
 818                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
 819         }
 820
 821         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 822 }
 823
 824 /**
 825  * iavf_restore_filters
 826  * @adapter: board private structure
 827  *
 828  * Restore existing non MAC filters when VF netdev comes back up
 829  **/
 830 static void iavf_restore_filters(struct iavf_adapter *adapter)
 831 {
 832         u16 vid;
 833
 834         /* re-add all VLAN filters */
 835         for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID)
 836                 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q));
 837
 838         for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID)
 839                 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD));
 840 }
 841
 842 /**
 843  * iavf_get_num_vlans_added - get number of VLANs added
 844  * @adapter: board private structure
 845  */
 846 static u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
 847 {
 848         return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) +
 849                 bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID);
 850 }
 851
 852 /**
 853  * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
 854  * @adapter: board private structure
 855  *
 856  * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
 857  * do not impose a limit as that maintains current behavior and for
 858  * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
 859  **/
 860 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
 861 {
 862         /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
 863          * never been a limit on the VF driver side
 864          */
 865         if (VLAN_ALLOWED(adapter))
 866                 return VLAN_N_VID;
 867         else if (VLAN_V2_ALLOWED(adapter))
 868                 return adapter->vlan_v2_caps.filtering.max_filters;
 869
 870         return 0;
 871 }
 872
 873 /**
 874  * iavf_max_vlans_added - check if maximum VLANs allowed already exist
 875  * @adapter: board private structure
 876  **/
 877 static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
 878 {
 879         if (iavf_get_num_vlans_added(adapter) <
 880             iavf_get_max_vlans_allowed(adapter))
 881                 return false;
 882
 883         return true;
 884 }
 885
 886 /**
 887  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
 888  * @netdev: network device struct
 889  * @proto: unused protocol data
 890  * @vid: VLAN tag
 891  **/
 892 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
 893                                 __always_unused __be16 proto, u16 vid)
 894 {
 895         struct iavf_adapter *adapter = netdev_priv(netdev);
 896
 897         if (!VLAN_FILTERING_ALLOWED(adapter))
 898                 return -EIO;
 899
 900         if (iavf_max_vlans_added(adapter)) {
 901                 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
 902                            iavf_get_max_vlans_allowed(adapter));
 903                 return -EIO;
 904         }
 905
 906         if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
 907                 return -ENOMEM;
 908
 909         if (proto == cpu_to_be16(ETH_P_8021Q))
 910                 set_bit(vid, adapter->vsi.active_cvlans);
 911         else
 912                 set_bit(vid, adapter->vsi.active_svlans);
 913
 914         return 0;
 915 }
 916
 917 /**
 918  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 919  * @netdev: network device struct
 920  * @proto: unused protocol data
 921  * @vid: VLAN tag
 922  **/
 923 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
 924                                  __always_unused __be16 proto, u16 vid)
 925 {
 926         struct iavf_adapter *adapter = netdev_priv(netdev);
 927
 928         iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
 929         if (proto == cpu_to_be16(ETH_P_8021Q))
 930                 clear_bit(vid, adapter->vsi.active_cvlans);
 931         else
 932                 clear_bit(vid, adapter->vsi.active_svlans);
 933
 934         return 0;
 935 }
 936
 937 /**
 938  * iavf_find_filter - Search filter list for specific mac filter
 939  * @adapter: board private structure
 940  * @macaddr: the MAC address
 941  *
 942  * Returns ptr to the filter object or NULL. Must be called while holding the
 943  * mac_vlan_list_lock.
 944  **/
 945 static struct
 946 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
 947                                   const u8 *macaddr)
 948 {
 949         struct iavf_mac_filter *f;
 950
 951         if (!macaddr)
 952                 return NULL;
 953
 954         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 955                 if (ether_addr_equal(macaddr, f->macaddr))
 956                         return f;
 957         }
 958         return NULL;
 959 }
 960
 961 /**
 962  * iavf_add_filter - Add a mac filter to the filter list
 963  * @adapter: board private structure
 964  * @macaddr: the MAC address
 965  *
 966  * Returns ptr to the filter object or NULL when no memory available.
 967  **/
 968 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
 969                                         const u8 *macaddr)
 970 {
 971         struct iavf_mac_filter *f;
 972
 973         if (!macaddr)
 974                 return NULL;
 975
 976         f = iavf_find_filter(adapter, macaddr);
 977         if (!f) {
 978                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 979                 if (!f)
 980                         return f;
 981
 982                 ether_addr_copy(f->macaddr, macaddr);
 983
 984                 list_add_tail(&f->list, &adapter->mac_filter_list);
 985                 f->add = true;
 986                 f->is_new_mac = true;
 987                 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
 988                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
 989         } else {
 990                 f->remove = false;
 991         }
 992
 993         return f;
 994 }
 995
 996 /**
 997  * iavf_set_mac - NDO callback to set port mac address
 998  * @netdev: network interface device structure
 999  * @p: pointer to an address structure
1000  *
1001  * Returns 0 on success, negative on failure
1002  **/
1003 static int iavf_set_mac(struct net_device *netdev, void *p)
1004 {
1005         struct iavf_adapter *adapter = netdev_priv(netdev);
1006         struct iavf_hw *hw = &adapter->hw;
1007         struct iavf_mac_filter *f;
1008         struct sockaddr *addr = p;
1009
1010         if (!is_valid_ether_addr(addr->sa_data))
1011                 return -EADDRNOTAVAIL;
1012
1013         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
1014                 return 0;
1015
1016         spin_lock_bh(&adapter->mac_vlan_list_lock);
1017
1018         f = iavf_find_filter(adapter, hw->mac.addr);
1019         if (f) {
1020                 f->remove = true;
1021                 f->is_primary = true;
1022                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1023         }
1024
1025         f = iavf_add_filter(adapter, addr->sa_data);
1026         if (f) {
1027                 f->is_primary = true;
1028                 ether_addr_copy(hw->mac.addr, addr->sa_data);
1029         }
1030
1031         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1032
1033         /* schedule the watchdog task to immediately process the request */
1034         if (f)
1035                 queue_work(iavf_wq, &adapter->watchdog_task.work);
1036
1037         return (f == NULL) ? -ENOMEM : 0;
1038 }
1039
1040 /**
1041  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1042  * @netdev: the netdevice
1043  * @addr: address to add
1044  *
1045  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1046  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1047  */
1048 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1049 {
1050         struct iavf_adapter *adapter = netdev_priv(netdev);
1051
1052         if (iavf_add_filter(adapter, addr))
1053                 return 0;
1054         else
1055                 return -ENOMEM;
1056 }
1057
1058 /**
1059  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1060  * @netdev: the netdevice
1061  * @addr: address to add
1062  *
1063  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1064  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1065  */
1066 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1067 {
1068         struct iavf_adapter *adapter = netdev_priv(netdev);
1069         struct iavf_mac_filter *f;
1070
1071         /* Under some circumstances, we might receive a request to delete
1072          * our own device address from our uc list. Because we store the
1073          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1074          * such requests and not delete our device address from this list.
1075          */
1076         if (ether_addr_equal(addr, netdev->dev_addr))
1077                 return 0;
1078
1079         f = iavf_find_filter(adapter, addr);
1080         if (f) {
1081                 f->remove = true;
1082                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1083         }
1084         return 0;
1085 }
1086
1087 /**
1088  * iavf_set_rx_mode - NDO callback to set the netdev filters
1089  * @netdev: network interface device structure
1090  **/
1091 static void iavf_set_rx_mode(struct net_device *netdev)
1092 {
1093         struct iavf_adapter *adapter = netdev_priv(netdev);
1094
1095         spin_lock_bh(&adapter->mac_vlan_list_lock);
1096         __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1097         __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1098         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1099
1100         if (netdev->flags & IFF_PROMISC &&
1101             !(adapter->flags & IAVF_FLAG_PROMISC_ON))
1102                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
1103         else if (!(netdev->flags & IFF_PROMISC) &&
1104                  adapter->flags & IAVF_FLAG_PROMISC_ON)
1105                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
1106
1107         if (netdev->flags & IFF_ALLMULTI &&
1108             !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
1109                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
1110         else if (!(netdev->flags & IFF_ALLMULTI) &&
1111                  adapter->flags & IAVF_FLAG_ALLMULTI_ON)
1112                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
1113 }
1114
1115 /**
1116  * iavf_napi_enable_all - enable NAPI on all queue vectors
1117  * @adapter: board private structure
1118  **/
1119 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1120 {
1121         int q_idx;
1122         struct iavf_q_vector *q_vector;
1123         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1124
1125         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1126                 struct napi_struct *napi;
1127
1128                 q_vector = &adapter->q_vectors[q_idx];
1129                 napi = &q_vector->napi;
1130                 napi_enable(napi);
1131         }
1132 }
1133
1134 /**
1135  * iavf_napi_disable_all - disable NAPI on all queue vectors
1136  * @adapter: board private structure
1137  **/
1138 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1139 {
1140         int q_idx;
1141         struct iavf_q_vector *q_vector;
1142         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1143
1144         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1145                 q_vector = &adapter->q_vectors[q_idx];
1146                 napi_disable(&q_vector->napi);
1147         }
1148 }
1149
1150 /**
1151  * iavf_configure - set up transmit and receive data structures
1152  * @adapter: board private structure
1153  **/
1154 static void iavf_configure(struct iavf_adapter *adapter)
1155 {
1156         struct net_device *netdev = adapter->netdev;
1157         int i;
1158
1159         iavf_set_rx_mode(netdev);
1160
1161         iavf_configure_tx(adapter);
1162         iavf_configure_rx(adapter);
1163         adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1164
1165         for (i = 0; i < adapter->num_active_queues; i++) {
1166                 struct iavf_ring *ring = &adapter->rx_rings[i];
1167
1168                 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
1169         }
1170 }
1171
1172 /**
1173  * iavf_up_complete - Finish the last steps of bringing up a connection
1174  * @adapter: board private structure
1175  *
1176  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1177  **/
1178 static void iavf_up_complete(struct iavf_adapter *adapter)
1179 {
1180         iavf_change_state(adapter, __IAVF_RUNNING);
1181         clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1182
1183         iavf_napi_enable_all(adapter);
1184
1185         adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1186         if (CLIENT_ENABLED(adapter))
1187                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
1188         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1189 }
1190
1191 /**
1192  * iavf_down - Shutdown the connection processing
1193  * @adapter: board private structure
1194  *
1195  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1196  **/
1197 void iavf_down(struct iavf_adapter *adapter)
1198 {
1199         struct net_device *netdev = adapter->netdev;
1200         struct iavf_vlan_filter *vlf;
1201         struct iavf_cloud_filter *cf;
1202         struct iavf_fdir_fltr *fdir;
1203         struct iavf_mac_filter *f;
1204         struct iavf_adv_rss *rss;
1205
1206         if (adapter->state <= __IAVF_DOWN_PENDING)
1207                 return;
1208
1209         netif_carrier_off(netdev);
1210         netif_tx_disable(netdev);
1211         adapter->link_up = false;
1212         iavf_napi_disable_all(adapter);
1213         iavf_irq_disable(adapter);
1214
1215         spin_lock_bh(&adapter->mac_vlan_list_lock);
1216
1217         /* clear the sync flag on all filters */
1218         __dev_uc_unsync(adapter->netdev, NULL);
1219         __dev_mc_unsync(adapter->netdev, NULL);
1220
1221         /* remove all MAC filters */
1222         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1223                 f->remove = true;
1224         }
1225
1226         /* remove all VLAN filters */
1227         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1228                 vlf->remove = true;
1229         }
1230
1231         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1232
1233         /* remove all cloud filters */
1234         spin_lock_bh(&adapter->cloud_filter_list_lock);
1235         list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1236                 cf->del = true;
1237         }
1238         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1239
1240         /* remove all Flow Director filters */
1241         spin_lock_bh(&adapter->fdir_fltr_lock);
1242         list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
1243                 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1244         }
1245         spin_unlock_bh(&adapter->fdir_fltr_lock);
1246
1247         /* remove all advance RSS configuration */
1248         spin_lock_bh(&adapter->adv_rss_lock);
1249         list_for_each_entry(rss, &adapter->adv_rss_list_head, list)
1250                 rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1251         spin_unlock_bh(&adapter->adv_rss_lock);
1252
1253         if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) {
1254                 /* cancel any current operation */
1255                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1256                 /* Schedule operations to close down the HW. Don't wait
1257                  * here for this to complete. The watchdog is still running
1258                  * and it will take care of this.
1259                  */
1260                 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER;
1261                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1262                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1263                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1264                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1265                 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1266         }
1267
1268         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1269 }
1270
1271 /**
1272  * iavf_acquire_msix_vectors - Setup the MSIX capability
1273  * @adapter: board private structure
1274  * @vectors: number of vectors to request
1275  *
1276  * Work with the OS to set up the MSIX vectors needed.
1277  *
1278  * Returns 0 on success, negative on failure
1279  **/
1280 static int
1281 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1282 {
1283         int err, vector_threshold;
1284
1285         /* We'll want at least 3 (vector_threshold):
1286          * 0) Other (Admin Queue and link, mostly)
1287          * 1) TxQ[0] Cleanup
1288          * 2) RxQ[0] Cleanup
1289          */
1290         vector_threshold = MIN_MSIX_COUNT;
1291
1292         /* The more we get, the more we will assign to Tx/Rx Cleanup
1293          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1294          * Right now, we simply care about how many we'll get; we'll
1295          * set them up later while requesting irq's.
1296          */
1297         err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1298                                     vector_threshold, vectors);
1299         if (err < 0) {
1300                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1301                 kfree(adapter->msix_entries);
1302                 adapter->msix_entries = NULL;
1303                 return err;
1304         }
1305
1306         /* Adjust for only the vectors we'll use, which is minimum
1307          * of max_msix_q_vectors + NONQ_VECS, or the number of
1308          * vectors we were allocated.
1309          */
1310         adapter->num_msix_vectors = err;
1311         return 0;
1312 }
1313
1314 /**
1315  * iavf_free_queues - Free memory for all rings
1316  * @adapter: board private structure to initialize
1317  *
1318  * Free all of the memory associated with queue pairs.
1319  **/
1320 static void iavf_free_queues(struct iavf_adapter *adapter)
1321 {
1322         if (!adapter->vsi_res)
1323                 return;
1324         adapter->num_active_queues = 0;
1325         kfree(adapter->tx_rings);
1326         adapter->tx_rings = NULL;
1327         kfree(adapter->rx_rings);
1328         adapter->rx_rings = NULL;
1329 }
1330
1331 /**
1332  * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1333  * @adapter: board private structure
1334  *
1335  * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1336  * stripped in certain descriptor fields. Instead of checking the offload
1337  * capability bits in the hot path, cache the location the ring specific
1338  * flags.
1339  */
1340 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1341 {
1342         int i;
1343
1344         for (i = 0; i < adapter->num_active_queues; i++) {
1345                 struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1346                 struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1347
1348                 /* prevent multiple L2TAG bits being set after VFR */
1349                 tx_ring->flags &=
1350                         ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1351                           IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1352                 rx_ring->flags &=
1353                         ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1354                           IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1355
1356                 if (VLAN_ALLOWED(adapter)) {
1357                         tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1358                         rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1359                 } else if (VLAN_V2_ALLOWED(adapter)) {
1360                         struct virtchnl_vlan_supported_caps *stripping_support;
1361                         struct virtchnl_vlan_supported_caps *insertion_support;
1362
1363                         stripping_support =
1364                                 &adapter->vlan_v2_caps.offloads.stripping_support;
1365                         insertion_support =
1366                                 &adapter->vlan_v2_caps.offloads.insertion_support;
1367
1368                         if (stripping_support->outer) {
1369                                 if (stripping_support->outer &
1370                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1371                                         rx_ring->flags |=
1372                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1373                                 else if (stripping_support->outer &
1374                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1375                                         rx_ring->flags |=
1376                                                 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1377                         } else if (stripping_support->inner) {
1378                                 if (stripping_support->inner &
1379                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1380                                         rx_ring->flags |=
1381                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1382                                 else if (stripping_support->inner &
1383                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1384                                         rx_ring->flags |=
1385                                                 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1386                         }
1387
1388                         if (insertion_support->outer) {
1389                                 if (insertion_support->outer &
1390                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1391                                         tx_ring->flags |=
1392                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1393                                 else if (insertion_support->outer &
1394                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1395                                         tx_ring->flags |=
1396                                                 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1397                         } else if (insertion_support->inner) {
1398                                 if (insertion_support->inner &
1399                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1400                                         tx_ring->flags |=
1401                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1402                                 else if (insertion_support->inner &
1403                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1404                                         tx_ring->flags |=
1405                                                 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1406                         }
1407                 }
1408         }
1409 }
1410
1411 /**
1412  * iavf_alloc_queues - Allocate memory for all rings
1413  * @adapter: board private structure to initialize
1414  *
1415  * We allocate one ring per queue at run-time since we don't know the
1416  * number of queues at compile-time.  The polling_netdev array is
1417  * intended for Multiqueue, but should work fine with a single queue.
1418  **/
1419 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1420 {
1421         int i, num_active_queues;
1422
1423         /* If we're in reset reallocating queues we don't actually know yet for
1424          * certain the PF gave us the number of queues we asked for but we'll
1425          * assume it did.  Once basic reset is finished we'll confirm once we
1426          * start negotiating config with PF.
1427          */
1428         if (adapter->num_req_queues)
1429                 num_active_queues = adapter->num_req_queues;
1430         else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1431                  adapter->num_tc)
1432                 num_active_queues = adapter->ch_config.total_qps;
1433         else
1434                 num_active_queues = min_t(int,
1435                                           adapter->vsi_res->num_queue_pairs,
1436                                           (int)(num_online_cpus()));
1437
1438
1439         adapter->tx_rings = kcalloc(num_active_queues,
1440                                     sizeof(struct iavf_ring), GFP_KERNEL);
1441         if (!adapter->tx_rings)
1442                 goto err_out;
1443         adapter->rx_rings = kcalloc(num_active_queues,
1444                                     sizeof(struct iavf_ring), GFP_KERNEL);
1445         if (!adapter->rx_rings)
1446                 goto err_out;
1447
1448         for (i = 0; i < num_active_queues; i++) {
1449                 struct iavf_ring *tx_ring;
1450                 struct iavf_ring *rx_ring;
1451
1452                 tx_ring = &adapter->tx_rings[i];
1453
1454                 tx_ring->queue_index = i;
1455                 tx_ring->netdev = adapter->netdev;
1456                 tx_ring->dev = &adapter->pdev->dev;
1457                 tx_ring->count = adapter->tx_desc_count;
1458                 tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1459                 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1460                         tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1461
1462                 rx_ring = &adapter->rx_rings[i];
1463                 rx_ring->queue_index = i;
1464                 rx_ring->netdev = adapter->netdev;
1465                 rx_ring->dev = &adapter->pdev->dev;
1466                 rx_ring->count = adapter->rx_desc_count;
1467                 rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1468         }
1469
1470         adapter->num_active_queues = num_active_queues;
1471
1472         iavf_set_queue_vlan_tag_loc(adapter);
1473
1474         return 0;
1475
1476 err_out:
1477         iavf_free_queues(adapter);
1478         return -ENOMEM;
1479 }
1480
1481 /**
1482  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1483  * @adapter: board private structure to initialize
1484  *
1485  * Attempt to configure the interrupts using the best available
1486  * capabilities of the hardware and the kernel.
1487  **/
1488 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1489 {
1490         int vector, v_budget;
1491         int pairs = 0;
1492         int err = 0;
1493
1494         if (!adapter->vsi_res) {
1495                 err = -EIO;
1496                 goto out;
1497         }
1498         pairs = adapter->num_active_queues;
1499
1500         /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1501          * us much good if we have more vectors than CPUs. However, we already
1502          * limit the total number of queues by the number of CPUs so we do not
1503          * need any further limiting here.
1504          */
1505         v_budget = min_t(int, pairs + NONQ_VECS,
1506                          (int)adapter->vf_res->max_vectors);
1507
1508         adapter->msix_entries = kcalloc(v_budget,
1509                                         sizeof(struct msix_entry), GFP_KERNEL);
1510         if (!adapter->msix_entries) {
1511                 err = -ENOMEM;
1512                 goto out;
1513         }
1514
1515         for (vector = 0; vector < v_budget; vector++)
1516                 adapter->msix_entries[vector].entry = vector;
1517
1518         err = iavf_acquire_msix_vectors(adapter, v_budget);
1519
1520 out:
1521         netif_set_real_num_rx_queues(adapter->netdev, pairs);
1522         netif_set_real_num_tx_queues(adapter->netdev, pairs);
1523         return err;
1524 }
1525
1526 /**
1527  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1528  * @adapter: board private structure
1529  *
1530  * Return 0 on success, negative on failure
1531  **/
1532 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1533 {
1534         struct iavf_aqc_get_set_rss_key_data *rss_key =
1535                 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1536         struct iavf_hw *hw = &adapter->hw;
1537         enum iavf_status status;
1538
1539         if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1540                 /* bail because we already have a command pending */
1541                 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1542                         adapter->current_op);
1543                 return -EBUSY;
1544         }
1545
1546         status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1547         if (status) {
1548                 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1549                         iavf_stat_str(hw, status),
1550                         iavf_aq_str(hw, hw->aq.asq_last_status));
1551                 return iavf_status_to_errno(status);
1552
1553         }
1554
1555         status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1556                                      adapter->rss_lut, adapter->rss_lut_size);
1557         if (status) {
1558                 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1559                         iavf_stat_str(hw, status),
1560                         iavf_aq_str(hw, hw->aq.asq_last_status));
1561                 return iavf_status_to_errno(status);
1562         }
1563
1564         return 0;
1565
1566 }
1567
1568 /**
1569  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1570  * @adapter: board private structure
1571  *
1572  * Returns 0 on success, negative on failure
1573  **/
1574 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1575 {
1576         struct iavf_hw *hw = &adapter->hw;
1577         u32 *dw;
1578         u16 i;
1579
1580         dw = (u32 *)adapter->rss_key;
1581         for (i = 0; i <= adapter->rss_key_size / 4; i++)
1582                 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1583
1584         dw = (u32 *)adapter->rss_lut;
1585         for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1586                 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1587
1588         iavf_flush(hw);
1589
1590         return 0;
1591 }
1592
1593 /**
1594  * iavf_config_rss - Configure RSS keys and lut
1595  * @adapter: board private structure
1596  *
1597  * Returns 0 on success, negative on failure
1598  **/
1599 int iavf_config_rss(struct iavf_adapter *adapter)
1600 {
1601
1602         if (RSS_PF(adapter)) {
1603                 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1604                                         IAVF_FLAG_AQ_SET_RSS_KEY;
1605                 return 0;
1606         } else if (RSS_AQ(adapter)) {
1607                 return iavf_config_rss_aq(adapter);
1608         } else {
1609                 return iavf_config_rss_reg(adapter);
1610         }
1611 }
1612
1613 /**
1614  * iavf_fill_rss_lut - Fill the lut with default values
1615  * @adapter: board private structure
1616  **/
1617 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1618 {
1619         u16 i;
1620
1621         for (i = 0; i < adapter->rss_lut_size; i++)
1622                 adapter->rss_lut[i] = i % adapter->num_active_queues;
1623 }
1624
1625 /**
1626  * iavf_init_rss - Prepare for RSS
1627  * @adapter: board private structure
1628  *
1629  * Return 0 on success, negative on failure
1630  **/
1631 static int iavf_init_rss(struct iavf_adapter *adapter)
1632 {
1633         struct iavf_hw *hw = &adapter->hw;
1634
1635         if (!RSS_PF(adapter)) {
1636                 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1637                 if (adapter->vf_res->vf_cap_flags &
1638                     VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1639                         adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1640                 else
1641                         adapter->hena = IAVF_DEFAULT_RSS_HENA;
1642
1643                 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1644                 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1645         }
1646
1647         iavf_fill_rss_lut(adapter);
1648         netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1649
1650         return iavf_config_rss(adapter);
1651 }
1652
1653 /**
1654  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1655  * @adapter: board private structure to initialize
1656  *
1657  * We allocate one q_vector per queue interrupt.  If allocation fails we
1658  * return -ENOMEM.
1659  **/
1660 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1661 {
1662         int q_idx = 0, num_q_vectors;
1663         struct iavf_q_vector *q_vector;
1664
1665         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1666         adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1667                                      GFP_KERNEL);
1668         if (!adapter->q_vectors)
1669                 return -ENOMEM;
1670
1671         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1672                 q_vector = &adapter->q_vectors[q_idx];
1673                 q_vector->adapter = adapter;
1674                 q_vector->vsi = &adapter->vsi;
1675                 q_vector->v_idx = q_idx;
1676                 q_vector->reg_idx = q_idx;
1677                 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1678                 netif_napi_add(adapter->netdev, &q_vector->napi,
1679                                iavf_napi_poll, NAPI_POLL_WEIGHT);
1680         }
1681
1682         return 0;
1683 }
1684
1685 /**
1686  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1687  * @adapter: board private structure to initialize
1688  *
1689  * This function frees the memory allocated to the q_vectors.  In addition if
1690  * NAPI is enabled it will delete any references to the NAPI struct prior
1691  * to freeing the q_vector.
1692  **/
1693 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1694 {
1695         int q_idx, num_q_vectors;
1696         int napi_vectors;
1697
1698         if (!adapter->q_vectors)
1699                 return;
1700
1701         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1702         napi_vectors = adapter->num_active_queues;
1703
1704         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1705                 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1706
1707                 if (q_idx < napi_vectors)
1708                         netif_napi_del(&q_vector->napi);
1709         }
1710         kfree(adapter->q_vectors);
1711         adapter->q_vectors = NULL;
1712 }
1713
1714 /**
1715  * iavf_reset_interrupt_capability - Reset MSIX setup
1716  * @adapter: board private structure
1717  *
1718  **/
1719 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1720 {
1721         if (!adapter->msix_entries)
1722                 return;
1723
1724         pci_disable_msix(adapter->pdev);
1725         kfree(adapter->msix_entries);
1726         adapter->msix_entries = NULL;
1727 }
1728
1729 /**
1730  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1731  * @adapter: board private structure to initialize
1732  *
1733  **/
1734 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1735 {
1736         int err;
1737
1738         err = iavf_alloc_queues(adapter);
1739         if (err) {
1740                 dev_err(&adapter->pdev->dev,
1741                         "Unable to allocate memory for queues\n");
1742                 goto err_alloc_queues;
1743         }
1744
1745         rtnl_lock();
1746         err = iavf_set_interrupt_capability(adapter);
1747         rtnl_unlock();
1748         if (err) {
1749                 dev_err(&adapter->pdev->dev,
1750                         "Unable to setup interrupt capabilities\n");
1751                 goto err_set_interrupt;
1752         }
1753
1754         err = iavf_alloc_q_vectors(adapter);
1755         if (err) {
1756                 dev_err(&adapter->pdev->dev,
1757                         "Unable to allocate memory for queue vectors\n");
1758                 goto err_alloc_q_vectors;
1759         }
1760
1761         /* If we've made it so far while ADq flag being ON, then we haven't
1762          * bailed out anywhere in middle. And ADq isn't just enabled but actual
1763          * resources have been allocated in the reset path.
1764          * Now we can truly claim that ADq is enabled.
1765          */
1766         if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1767             adapter->num_tc)
1768                 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1769                          adapter->num_tc);
1770
1771         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1772                  (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1773                  adapter->num_active_queues);
1774
1775         return 0;
1776 err_alloc_q_vectors:
1777         iavf_reset_interrupt_capability(adapter);
1778 err_set_interrupt:
1779         iavf_free_queues(adapter);
1780 err_alloc_queues:
1781         return err;
1782 }
1783
1784 /**
1785  * iavf_free_rss - Free memory used by RSS structs
1786  * @adapter: board private structure
1787  **/
1788 static void iavf_free_rss(struct iavf_adapter *adapter)
1789 {
1790         kfree(adapter->rss_key);
1791         adapter->rss_key = NULL;
1792
1793         kfree(adapter->rss_lut);
1794         adapter->rss_lut = NULL;
1795 }
1796
1797 /**
1798  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1799  * @adapter: board private structure
1800  *
1801  * Returns 0 on success, negative on failure
1802  **/
1803 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1804 {
1805         struct net_device *netdev = adapter->netdev;
1806         int err;
1807
1808         if (netif_running(netdev))
1809                 iavf_free_traffic_irqs(adapter);
1810         iavf_free_misc_irq(adapter);
1811         iavf_reset_interrupt_capability(adapter);
1812         iavf_free_q_vectors(adapter);
1813         iavf_free_queues(adapter);
1814
1815         err =  iavf_init_interrupt_scheme(adapter);
1816         if (err)
1817                 goto err;
1818
1819         netif_tx_stop_all_queues(netdev);
1820
1821         err = iavf_request_misc_irq(adapter);
1822         if (err)
1823                 goto err;
1824
1825         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1826
1827         iavf_map_rings_to_vectors(adapter);
1828 err:
1829         return err;
1830 }
1831
1832 /**
1833  * iavf_process_aq_command - process aq_required flags
1834  * and sends aq command
1835  * @adapter: pointer to iavf adapter structure
1836  *
1837  * Returns 0 on success
1838  * Returns error code if no command was sent
1839  * or error code if the command failed.
1840  **/
1841 static int iavf_process_aq_command(struct iavf_adapter *adapter)
1842 {
1843         if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
1844                 return iavf_send_vf_config_msg(adapter);
1845         if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
1846                 return iavf_send_vf_offload_vlan_v2_msg(adapter);
1847         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
1848                 iavf_disable_queues(adapter);
1849                 return 0;
1850         }
1851
1852         if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
1853                 iavf_map_queues(adapter);
1854                 return 0;
1855         }
1856
1857         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
1858                 iavf_add_ether_addrs(adapter);
1859                 return 0;
1860         }
1861
1862         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
1863                 iavf_add_vlans(adapter);
1864                 return 0;
1865         }
1866
1867         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
1868                 iavf_del_ether_addrs(adapter);
1869                 return 0;
1870         }
1871
1872         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
1873                 iavf_del_vlans(adapter);
1874                 return 0;
1875         }
1876
1877         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
1878                 iavf_enable_vlan_stripping(adapter);
1879                 return 0;
1880         }
1881
1882         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
1883                 iavf_disable_vlan_stripping(adapter);
1884                 return 0;
1885         }
1886
1887         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
1888                 iavf_configure_queues(adapter);
1889                 return 0;
1890         }
1891
1892         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
1893                 iavf_enable_queues(adapter);
1894                 return 0;
1895         }
1896
1897         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
1898                 /* This message goes straight to the firmware, not the
1899                  * PF, so we don't have to set current_op as we will
1900                  * not get a response through the ARQ.
1901                  */
1902                 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
1903                 return 0;
1904         }
1905         if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
1906                 iavf_get_hena(adapter);
1907                 return 0;
1908         }
1909         if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
1910                 iavf_set_hena(adapter);
1911                 return 0;
1912         }
1913         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
1914                 iavf_set_rss_key(adapter);
1915                 return 0;
1916         }
1917         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
1918                 iavf_set_rss_lut(adapter);
1919                 return 0;
1920         }
1921
1922         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
1923                 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
1924                                        FLAG_VF_MULTICAST_PROMISC);
1925                 return 0;
1926         }
1927
1928         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
1929                 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
1930                 return 0;
1931         }
1932         if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) ||
1933             (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
1934                 iavf_set_promiscuous(adapter, 0);
1935                 return 0;
1936         }
1937
1938         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
1939                 iavf_enable_channels(adapter);
1940                 return 0;
1941         }
1942
1943         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
1944                 iavf_disable_channels(adapter);
1945                 return 0;
1946         }
1947         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1948                 iavf_add_cloud_filter(adapter);
1949                 return 0;
1950         }
1951
1952         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1953                 iavf_del_cloud_filter(adapter);
1954                 return 0;
1955         }
1956         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1957                 iavf_del_cloud_filter(adapter);
1958                 return 0;
1959         }
1960         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1961                 iavf_add_cloud_filter(adapter);
1962                 return 0;
1963         }
1964         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
1965                 iavf_add_fdir_filter(adapter);
1966                 return IAVF_SUCCESS;
1967         }
1968         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
1969                 iavf_del_fdir_filter(adapter);
1970                 return IAVF_SUCCESS;
1971         }
1972         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
1973                 iavf_add_adv_rss_cfg(adapter);
1974                 return 0;
1975         }
1976         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
1977                 iavf_del_adv_rss_cfg(adapter);
1978                 return 0;
1979         }
1980         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
1981                 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
1982                 return 0;
1983         }
1984         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
1985                 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
1986                 return 0;
1987         }
1988         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
1989                 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
1990                 return 0;
1991         }
1992         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
1993                 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
1994                 return 0;
1995         }
1996         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
1997                 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
1998                 return 0;
1999         }
2000         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2001                 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2002                 return 0;
2003         }
2004         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2005                 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2006                 return 0;
2007         }
2008         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2009                 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2010                 return 0;
2011         }
2012
2013         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2014                 iavf_request_stats(adapter);
2015                 return 0;
2016         }
2017
2018         return -EAGAIN;
2019 }
2020
2021 /**
2022  * iavf_set_vlan_offload_features - set VLAN offload configuration
2023  * @adapter: board private structure
2024  * @prev_features: previous features used for comparison
2025  * @features: updated features used for configuration
2026  *
2027  * Set the aq_required bit(s) based on the requested features passed in to
2028  * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2029  * the watchdog if any changes are requested to expedite the request via
2030  * virtchnl.
2031  **/
2032 void
2033 iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2034                                netdev_features_t prev_features,
2035                                netdev_features_t features)
2036 {
2037         bool enable_stripping = true, enable_insertion = true;
2038         u16 vlan_ethertype = 0;
2039         u64 aq_required = 0;
2040
2041         /* keep cases separate because one ethertype for offloads can be
2042          * disabled at the same time as another is disabled, so check for an
2043          * enabled ethertype first, then check for disabled. Default to
2044          * ETH_P_8021Q so an ethertype is specified if disabling insertion and
2045          * stripping.
2046          */
2047         if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2048                 vlan_ethertype = ETH_P_8021AD;
2049         else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2050                 vlan_ethertype = ETH_P_8021Q;
2051         else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2052                 vlan_ethertype = ETH_P_8021AD;
2053         else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2054                 vlan_ethertype = ETH_P_8021Q;
2055         else
2056                 vlan_ethertype = ETH_P_8021Q;
2057
2058         if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2059                 enable_stripping = false;
2060         if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2061                 enable_insertion = false;
2062
2063         if (VLAN_ALLOWED(adapter)) {
2064                 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2065                  * stripping via virtchnl. VLAN insertion can be toggled on the
2066                  * netdev, but it doesn't require a virtchnl message
2067                  */
2068                 if (enable_stripping)
2069                         aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2070                 else
2071                         aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2072
2073         } else if (VLAN_V2_ALLOWED(adapter)) {
2074                 switch (vlan_ethertype) {
2075                 case ETH_P_8021Q:
2076                         if (enable_stripping)
2077                                 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2078                         else
2079                                 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2080
2081                         if (enable_insertion)
2082                                 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2083                         else
2084                                 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2085                         break;
2086                 case ETH_P_8021AD:
2087                         if (enable_stripping)
2088                                 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2089                         else
2090                                 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2091
2092                         if (enable_insertion)
2093                                 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2094                         else
2095                                 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2096                         break;
2097                 }
2098         }
2099
2100         if (aq_required) {
2101                 adapter->aq_required |= aq_required;
2102                 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
2103         }
2104 }
2105
2106 /**
2107  * iavf_startup - first step of driver startup
2108  * @adapter: board private structure
2109  *
2110  * Function process __IAVF_STARTUP driver state.
2111  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
2112  * when fails the state is changed to __IAVF_INIT_FAILED
2113  **/
2114 static void iavf_startup(struct iavf_adapter *adapter)
2115 {
2116         struct pci_dev *pdev = adapter->pdev;
2117         struct iavf_hw *hw = &adapter->hw;
2118         enum iavf_status status;
2119         int ret;
2120
2121         WARN_ON(adapter->state != __IAVF_STARTUP);
2122
2123         /* driver loaded, probe complete */
2124         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2125         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2126         status = iavf_set_mac_type(hw);
2127         if (status) {
2128                 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status);
2129                 goto err;
2130         }
2131
2132         ret = iavf_check_reset_complete(hw);
2133         if (ret) {
2134                 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2135                          ret);
2136                 goto err;
2137         }
2138         hw->aq.num_arq_entries = IAVF_AQ_LEN;
2139         hw->aq.num_asq_entries = IAVF_AQ_LEN;
2140         hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2141         hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2142
2143         status = iavf_init_adminq(hw);
2144         if (status) {
2145                 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2146                         status);
2147                 goto err;
2148         }
2149         ret = iavf_send_api_ver(adapter);
2150         if (ret) {
2151                 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2152                 iavf_shutdown_adminq(hw);
2153                 goto err;
2154         }
2155         iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
2156         return;
2157 err:
2158         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2159 }
2160
2161 /**
2162  * iavf_init_version_check - second step of driver startup
2163  * @adapter: board private structure
2164  *
2165  * Function process __IAVF_INIT_VERSION_CHECK driver state.
2166  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
2167  * when fails the state is changed to __IAVF_INIT_FAILED
2168  **/
2169 static void iavf_init_version_check(struct iavf_adapter *adapter)
2170 {
2171         struct pci_dev *pdev = adapter->pdev;
2172         struct iavf_hw *hw = &adapter->hw;
2173         int err = -EAGAIN;
2174
2175         WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2176
2177         if (!iavf_asq_done(hw)) {
2178                 dev_err(&pdev->dev, "Admin queue command never completed\n");
2179                 iavf_shutdown_adminq(hw);
2180                 iavf_change_state(adapter, __IAVF_STARTUP);
2181                 goto err;
2182         }
2183
2184         /* aq msg sent, awaiting reply */
2185         err = iavf_verify_api_ver(adapter);
2186         if (err) {
2187                 if (err == -EALREADY)
2188                         err = iavf_send_api_ver(adapter);
2189                 else
2190                         dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2191                                 adapter->pf_version.major,
2192                                 adapter->pf_version.minor,
2193                                 VIRTCHNL_VERSION_MAJOR,
2194                                 VIRTCHNL_VERSION_MINOR);
2195                 goto err;
2196         }
2197         err = iavf_send_vf_config_msg(adapter);
2198         if (err) {
2199                 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2200                         err);
2201                 goto err;
2202         }
2203         iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
2204         return;
2205 err:
2206         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2207 }
2208
2209 /**
2210  * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2211  * @adapter: board private structure
2212  */
2213 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2214 {
2215         int i, num_req_queues = adapter->num_req_queues;
2216         struct iavf_vsi *vsi = &adapter->vsi;
2217
2218         for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2219                 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2220                         adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2221         }
2222         if (!adapter->vsi_res) {
2223                 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2224                 return -ENODEV;
2225         }
2226
2227         if (num_req_queues &&
2228             num_req_queues > adapter->vsi_res->num_queue_pairs) {
2229                 /* Problem.  The PF gave us fewer queues than what we had
2230                  * negotiated in our request.  Need a reset to see if we can't
2231                  * get back to a working state.
2232                  */
2233                 dev_err(&adapter->pdev->dev,
2234                         "Requested %d queues, but PF only gave us %d.\n",
2235                         num_req_queues,
2236                         adapter->vsi_res->num_queue_pairs);
2237                 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2238                 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2239                 iavf_schedule_reset(adapter);
2240
2241                 return -EAGAIN;
2242         }
2243         adapter->num_req_queues = 0;
2244         adapter->vsi.id = adapter->vsi_res->vsi_id;
2245
2246         adapter->vsi.back = adapter;
2247         adapter->vsi.base_vector = 1;
2248         adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK;
2249         vsi->netdev = adapter->netdev;
2250         vsi->qs_handle = adapter->vsi_res->qset_handle;
2251         if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2252                 adapter->rss_key_size = adapter->vf_res->rss_key_size;
2253                 adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2254         } else {
2255                 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2256                 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2257         }
2258
2259         return 0;
2260 }
2261
2262 /**
2263  * iavf_init_get_resources - third step of driver startup
2264  * @adapter: board private structure
2265  *
2266  * Function process __IAVF_INIT_GET_RESOURCES driver state and
2267  * finishes driver initialization procedure.
2268  * When success the state is changed to __IAVF_DOWN
2269  * when fails the state is changed to __IAVF_INIT_FAILED
2270  **/
2271 static void iavf_init_get_resources(struct iavf_adapter *adapter)
2272 {
2273         struct pci_dev *pdev = adapter->pdev;
2274         struct iavf_hw *hw = &adapter->hw;
2275         int err;
2276
2277         WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2278         /* aq msg sent, awaiting reply */
2279         if (!adapter->vf_res) {
2280                 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2281                                           GFP_KERNEL);
2282                 if (!adapter->vf_res) {
2283                         err = -ENOMEM;
2284                         goto err;
2285                 }
2286         }
2287         err = iavf_get_vf_config(adapter);
2288         if (err == -EALREADY) {
2289                 err = iavf_send_vf_config_msg(adapter);
2290                 goto err_alloc;
2291         } else if (err == -EINVAL) {
2292                 /* We only get -EINVAL if the device is in a very bad
2293                  * state or if we've been disabled for previous bad
2294                  * behavior. Either way, we're done now.
2295                  */
2296                 iavf_shutdown_adminq(hw);
2297                 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2298                 return;
2299         }
2300         if (err) {
2301                 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2302                 goto err_alloc;
2303         }
2304
2305         err = iavf_parse_vf_resource_msg(adapter);
2306         if (err) {
2307                 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2308                         err);
2309                 goto err_alloc;
2310         }
2311         /* Some features require additional messages to negotiate extended
2312          * capabilities. These are processed in sequence by the
2313          * __IAVF_INIT_EXTENDED_CAPS driver state.
2314          */
2315         adapter->extended_caps = IAVF_EXTENDED_CAPS;
2316
2317         iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
2318         return;
2319
2320 err_alloc:
2321         kfree(adapter->vf_res);
2322         adapter->vf_res = NULL;
2323 err:
2324         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2325 }
2326
2327 /**
2328  * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2329  * @adapter: board private structure
2330  *
2331  * Function processes send of the extended VLAN V2 capability message to the
2332  * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2333  * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2334  */
2335 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2336 {
2337         int ret;
2338
2339         WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2340
2341         ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2342         if (ret && ret == -EOPNOTSUPP) {
2343                 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2344                  * we did not send the capability exchange message and do not
2345                  * expect a response.
2346                  */
2347                 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2348         }
2349
2350         /* We sent the message, so move on to the next step */
2351         adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2352 }
2353
2354 /**
2355  * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2356  * @adapter: board private structure
2357  *
2358  * Function processes receipt of the extended VLAN V2 capability message from
2359  * the PF.
2360  **/
2361 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2362 {
2363         int ret;
2364
2365         WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2366
2367         memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2368
2369         ret = iavf_get_vf_vlan_v2_caps(adapter);
2370         if (ret)
2371                 goto err;
2372
2373         /* We've processed receipt of the VLAN V2 caps message */
2374         adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2375         return;
2376 err:
2377         /* We didn't receive a reply. Make sure we try sending again when
2378          * __IAVF_INIT_FAILED attempts to recover.
2379          */
2380         adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2381         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2382 }
2383
2384 /**
2385  * iavf_init_process_extended_caps - Part of driver startup
2386  * @adapter: board private structure
2387  *
2388  * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2389  * handles negotiating capabilities for features which require an additional
2390  * message.
2391  *
2392  * Once all extended capabilities exchanges are finished, the driver will
2393  * transition into __IAVF_INIT_CONFIG_ADAPTER.
2394  */
2395 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2396 {
2397         WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2398
2399         /* Process capability exchange for VLAN V2 */
2400         if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2401                 iavf_init_send_offload_vlan_v2_caps(adapter);
2402                 return;
2403         } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2404                 iavf_init_recv_offload_vlan_v2_caps(adapter);
2405                 return;
2406         }
2407
2408         /* When we reach here, no further extended capabilities exchanges are
2409          * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2410          */
2411         iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
2412 }
2413
2414 /**
2415  * iavf_init_config_adapter - last part of driver startup
2416  * @adapter: board private structure
2417  *
2418  * After all the supported capabilities are negotiated, then the
2419  * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2420  */
2421 static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2422 {
2423         struct net_device *netdev = adapter->netdev;
2424         struct pci_dev *pdev = adapter->pdev;
2425         int err;
2426
2427         WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2428
2429         if (iavf_process_config(adapter))
2430                 goto err;
2431
2432         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2433
2434         adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2435
2436         netdev->netdev_ops = &iavf_netdev_ops;
2437         iavf_set_ethtool_ops(netdev);
2438         netdev->watchdog_timeo = 5 * HZ;
2439
2440         /* MTU range: 68 - 9710 */
2441         netdev->min_mtu = ETH_MIN_MTU;
2442         netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
2443
2444         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2445                 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2446                          adapter->hw.mac.addr);
2447                 eth_hw_addr_random(netdev);
2448                 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
2449         } else {
2450                 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2451                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2452         }
2453
2454         adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2455         adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2456         err = iavf_init_interrupt_scheme(adapter);
2457         if (err)
2458                 goto err_sw_init;
2459         iavf_map_rings_to_vectors(adapter);
2460         if (adapter->vf_res->vf_cap_flags &
2461                 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2462                 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2463
2464         err = iavf_request_misc_irq(adapter);
2465         if (err)
2466                 goto err_sw_init;
2467
2468         netif_carrier_off(netdev);
2469         adapter->link_up = false;
2470
2471         /* set the semaphore to prevent any callbacks after device registration
2472          * up to time when state of driver will be set to __IAVF_DOWN
2473          */
2474         rtnl_lock();
2475         if (!adapter->netdev_registered) {
2476                 err = register_netdevice(netdev);
2477                 if (err) {
2478                         rtnl_unlock();
2479                         goto err_register;
2480                 }
2481         }
2482
2483         adapter->netdev_registered = true;
2484
2485         netif_tx_stop_all_queues(netdev);
2486         if (CLIENT_ALLOWED(adapter)) {
2487                 err = iavf_lan_add_device(adapter);
2488                 if (err)
2489                         dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
2490                                  err);
2491         }
2492         dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2493         if (netdev->features & NETIF_F_GRO)
2494                 dev_info(&pdev->dev, "GRO is enabled\n");
2495
2496         iavf_change_state(adapter, __IAVF_DOWN);
2497         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2498         rtnl_unlock();
2499
2500         iavf_misc_irq_enable(adapter);
2501         wake_up(&adapter->down_waitqueue);
2502
2503         adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
2504         adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
2505         if (!adapter->rss_key || !adapter->rss_lut) {
2506                 err = -ENOMEM;
2507                 goto err_mem;
2508         }
2509         if (RSS_AQ(adapter))
2510                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2511         else
2512                 iavf_init_rss(adapter);
2513
2514         if (VLAN_V2_ALLOWED(adapter))
2515                 /* request initial VLAN offload settings */
2516                 iavf_set_vlan_offload_features(adapter, 0, netdev->features);
2517
2518         return;
2519 err_mem:
2520         iavf_free_rss(adapter);
2521 err_register:
2522         iavf_free_misc_irq(adapter);
2523 err_sw_init:
2524         iavf_reset_interrupt_capability(adapter);
2525 err:
2526         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2527 }
2528
2529 /**
2530  * iavf_watchdog_task - Periodic call-back task
2531  * @work: pointer to work_struct
2532  **/
2533 static void iavf_watchdog_task(struct work_struct *work)
2534 {
2535         struct iavf_adapter *adapter = container_of(work,
2536                                                     struct iavf_adapter,
2537                                                     watchdog_task.work);
2538         struct iavf_hw *hw = &adapter->hw;
2539         u32 reg_val;
2540
2541         if (!mutex_trylock(&adapter->crit_lock)) {
2542                 if (adapter->state == __IAVF_REMOVE)
2543                         return;
2544
2545                 goto restart_watchdog;
2546         }
2547
2548         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2549                 iavf_change_state(adapter, __IAVF_COMM_FAILED);
2550
2551         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2552                 adapter->aq_required = 0;
2553                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2554                 mutex_unlock(&adapter->crit_lock);
2555                 queue_work(iavf_wq, &adapter->reset_task);
2556                 return;
2557         }
2558
2559         switch (adapter->state) {
2560         case __IAVF_STARTUP:
2561                 iavf_startup(adapter);
2562                 mutex_unlock(&adapter->crit_lock);
2563                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2564                                    msecs_to_jiffies(30));
2565                 return;
2566         case __IAVF_INIT_VERSION_CHECK:
2567                 iavf_init_version_check(adapter);
2568                 mutex_unlock(&adapter->crit_lock);
2569                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2570                                    msecs_to_jiffies(30));
2571                 return;
2572         case __IAVF_INIT_GET_RESOURCES:
2573                 iavf_init_get_resources(adapter);
2574                 mutex_unlock(&adapter->crit_lock);
2575                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2576                                    msecs_to_jiffies(1));
2577                 return;
2578         case __IAVF_INIT_EXTENDED_CAPS:
2579                 iavf_init_process_extended_caps(adapter);
2580                 mutex_unlock(&adapter->crit_lock);
2581                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2582                                    msecs_to_jiffies(1));
2583                 return;
2584         case __IAVF_INIT_CONFIG_ADAPTER:
2585                 iavf_init_config_adapter(adapter);
2586                 mutex_unlock(&adapter->crit_lock);
2587                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2588                                    msecs_to_jiffies(1));
2589                 return;
2590         case __IAVF_INIT_FAILED:
2591                 if (test_bit(__IAVF_IN_REMOVE_TASK,
2592                              &adapter->crit_section)) {
2593                         /* Do not update the state and do not reschedule
2594                          * watchdog task, iavf_remove should handle this state
2595                          * as it can loop forever
2596                          */
2597                         mutex_unlock(&adapter->crit_lock);
2598                         return;
2599                 }
2600                 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2601                         dev_err(&adapter->pdev->dev,
2602                                 "Failed to communicate with PF; waiting before retry\n");
2603                         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2604                         iavf_shutdown_adminq(hw);
2605                         mutex_unlock(&adapter->crit_lock);
2606                         queue_delayed_work(iavf_wq,
2607                                            &adapter->watchdog_task, (5 * HZ));
2608                         return;
2609                 }
2610                 /* Try again from failed step*/
2611                 iavf_change_state(adapter, adapter->last_state);
2612                 mutex_unlock(&adapter->crit_lock);
2613                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ);
2614                 return;
2615         case __IAVF_COMM_FAILED:
2616                 if (test_bit(__IAVF_IN_REMOVE_TASK,
2617                              &adapter->crit_section)) {
2618                         /* Set state to __IAVF_INIT_FAILED and perform remove
2619                          * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2620                          * doesn't bring the state back to __IAVF_COMM_FAILED.
2621                          */
2622                         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2623                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2624                         mutex_unlock(&adapter->crit_lock);
2625                         return;
2626                 }
2627                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2628                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2629                 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2630                     reg_val == VIRTCHNL_VFR_COMPLETED) {
2631                         /* A chance for redemption! */
2632                         dev_err(&adapter->pdev->dev,
2633                                 "Hardware came out of reset. Attempting reinit.\n");
2634                         /* When init task contacts the PF and
2635                          * gets everything set up again, it'll restart the
2636                          * watchdog for us. Down, boy. Sit. Stay. Woof.
2637                          */
2638                         iavf_change_state(adapter, __IAVF_STARTUP);
2639                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2640                 }
2641                 adapter->aq_required = 0;
2642                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2643                 mutex_unlock(&adapter->crit_lock);
2644                 queue_delayed_work(iavf_wq,
2645                                    &adapter->watchdog_task,
2646                                    msecs_to_jiffies(10));
2647                 return;
2648         case __IAVF_RESETTING:
2649                 mutex_unlock(&adapter->crit_lock);
2650                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2651                 return;
2652         case __IAVF_DOWN:
2653         case __IAVF_DOWN_PENDING:
2654         case __IAVF_TESTING:
2655         case __IAVF_RUNNING:
2656                 if (adapter->current_op) {
2657                         if (!iavf_asq_done(hw)) {
2658                                 dev_dbg(&adapter->pdev->dev,
2659                                         "Admin queue timeout\n");
2660                                 iavf_send_api_ver(adapter);
2661                         }
2662                 } else {
2663                         int ret = iavf_process_aq_command(adapter);
2664
2665                         /* An error will be returned if no commands were
2666                          * processed; use this opportunity to update stats
2667                          * if the error isn't -ENOTSUPP
2668                          */
2669                         if (ret && ret != -EOPNOTSUPP &&
2670                             adapter->state == __IAVF_RUNNING)
2671                                 iavf_request_stats(adapter);
2672                 }
2673                 if (adapter->state == __IAVF_RUNNING)
2674                         iavf_detect_recover_hung(&adapter->vsi);
2675                 break;
2676         case __IAVF_REMOVE:
2677         default:
2678                 mutex_unlock(&adapter->crit_lock);
2679                 return;
2680         }
2681
2682         /* check for hw reset */
2683         reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2684         if (!reg_val) {
2685                 adapter->flags |= IAVF_FLAG_RESET_PENDING;
2686                 adapter->aq_required = 0;
2687                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2688                 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2689                 queue_work(iavf_wq, &adapter->reset_task);
2690                 mutex_unlock(&adapter->crit_lock);
2691                 queue_delayed_work(iavf_wq,
2692                                    &adapter->watchdog_task, HZ * 2);
2693                 return;
2694         }
2695
2696         schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2697         mutex_unlock(&adapter->crit_lock);
2698 restart_watchdog:
2699         if (adapter->state >= __IAVF_DOWN)
2700                 queue_work(iavf_wq, &adapter->adminq_task);
2701         if (adapter->aq_required)
2702                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2703                                    msecs_to_jiffies(20));
2704         else
2705                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2706 }
2707
2708 /**
2709  * iavf_disable_vf - disable VF
2710  * @adapter: board private structure
2711  *
2712  * Set communication failed flag and free all resources.
2713  * NOTE: This function is expected to be called with crit_lock being held.
2714  **/
2715 static void iavf_disable_vf(struct iavf_adapter *adapter)
2716 {
2717         struct iavf_mac_filter *f, *ftmp;
2718         struct iavf_vlan_filter *fv, *fvtmp;
2719         struct iavf_cloud_filter *cf, *cftmp;
2720
2721         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2722
2723         /* We don't use netif_running() because it may be true prior to
2724          * ndo_open() returning, so we can't assume it means all our open
2725          * tasks have finished, since we're not holding the rtnl_lock here.
2726          */
2727         if (adapter->state == __IAVF_RUNNING) {
2728                 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2729                 netif_carrier_off(adapter->netdev);
2730                 netif_tx_disable(adapter->netdev);
2731                 adapter->link_up = false;
2732                 iavf_napi_disable_all(adapter);
2733                 iavf_irq_disable(adapter);
2734                 iavf_free_traffic_irqs(adapter);
2735                 iavf_free_all_tx_resources(adapter);
2736                 iavf_free_all_rx_resources(adapter);
2737         }
2738
2739         spin_lock_bh(&adapter->mac_vlan_list_lock);
2740
2741         /* Delete all of the filters */
2742         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2743                 list_del(&f->list);
2744                 kfree(f);
2745         }
2746
2747         list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2748                 list_del(&fv->list);
2749                 kfree(fv);
2750         }
2751
2752         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2753
2754         spin_lock_bh(&adapter->cloud_filter_list_lock);
2755         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2756                 list_del(&cf->list);
2757                 kfree(cf);
2758                 adapter->num_cloud_filters--;
2759         }
2760         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2761
2762         iavf_free_misc_irq(adapter);
2763         iavf_reset_interrupt_capability(adapter);
2764         iavf_free_q_vectors(adapter);
2765         iavf_free_queues(adapter);
2766         memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2767         iavf_shutdown_adminq(&adapter->hw);
2768         adapter->netdev->flags &= ~IFF_UP;
2769         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2770         iavf_change_state(adapter, __IAVF_DOWN);
2771         wake_up(&adapter->down_waitqueue);
2772         dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2773 }
2774
2775 /**
2776  * iavf_reset_task - Call-back task to handle hardware reset
2777  * @work: pointer to work_struct
2778  *
2779  * During reset we need to shut down and reinitialize the admin queue
2780  * before we can use it to communicate with the PF again. We also clear
2781  * and reinit the rings because that context is lost as well.
2782  **/
2783 static void iavf_reset_task(struct work_struct *work)
2784 {
2785         struct iavf_adapter *adapter = container_of(work,
2786                                                       struct iavf_adapter,
2787                                                       reset_task);
2788         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2789         struct net_device *netdev = adapter->netdev;
2790         struct iavf_hw *hw = &adapter->hw;
2791         struct iavf_mac_filter *f, *ftmp;
2792         struct iavf_cloud_filter *cf;
2793         enum iavf_status status;
2794         u32 reg_val;
2795         int i = 0, err;
2796         bool running;
2797
2798         /* When device is being removed it doesn't make sense to run the reset
2799          * task, just return in such a case.
2800          */
2801         if (!mutex_trylock(&adapter->crit_lock)) {
2802                 if (adapter->state != __IAVF_REMOVE)
2803                         queue_work(iavf_wq, &adapter->reset_task);
2804
2805                 return;
2806         }
2807
2808         while (!mutex_trylock(&adapter->client_lock))
2809                 usleep_range(500, 1000);
2810         if (CLIENT_ENABLED(adapter)) {
2811                 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
2812                                     IAVF_FLAG_CLIENT_NEEDS_CLOSE |
2813                                     IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
2814                                     IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
2815                 cancel_delayed_work_sync(&adapter->client_task);
2816                 iavf_notify_client_close(&adapter->vsi, true);
2817         }
2818         iavf_misc_irq_disable(adapter);
2819         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2820                 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
2821                 /* Restart the AQ here. If we have been reset but didn't
2822                  * detect it, or if the PF had to reinit, our AQ will be hosed.
2823                  */
2824                 iavf_shutdown_adminq(hw);
2825                 iavf_init_adminq(hw);
2826                 iavf_request_reset(adapter);
2827         }
2828         adapter->flags |= IAVF_FLAG_RESET_PENDING;
2829
2830         /* poll until we see the reset actually happen */
2831         for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
2832                 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
2833                           IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2834                 if (!reg_val)
2835                         break;
2836                 usleep_range(5000, 10000);
2837         }
2838         if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
2839                 dev_info(&adapter->pdev->dev, "Never saw reset\n");
2840                 goto continue_reset; /* act like the reset happened */
2841         }
2842
2843         /* wait until the reset is complete and the PF is responding to us */
2844         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
2845                 /* sleep first to make sure a minimum wait time is met */
2846                 msleep(IAVF_RESET_WAIT_MS);
2847
2848                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2849                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2850                 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
2851                         break;
2852         }
2853
2854         pci_set_master(adapter->pdev);
2855         pci_restore_msi_state(adapter->pdev);
2856
2857         if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
2858                 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
2859                         reg_val);
2860                 iavf_disable_vf(adapter);
2861                 mutex_unlock(&adapter->client_lock);
2862                 mutex_unlock(&adapter->crit_lock);
2863                 return; /* Do not attempt to reinit. It's dead, Jim. */
2864         }
2865
2866 continue_reset:
2867         /* We don't use netif_running() because it may be true prior to
2868          * ndo_open() returning, so we can't assume it means all our open
2869          * tasks have finished, since we're not holding the rtnl_lock here.
2870          */
2871         running = adapter->state == __IAVF_RUNNING;
2872
2873         if (running) {
2874                 netif_carrier_off(netdev);
2875                 netif_tx_stop_all_queues(netdev);
2876                 adapter->link_up = false;
2877                 iavf_napi_disable_all(adapter);
2878         }
2879         iavf_irq_disable(adapter);
2880
2881         iavf_change_state(adapter, __IAVF_RESETTING);
2882         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2883
2884         /* free the Tx/Rx rings and descriptors, might be better to just
2885          * re-use them sometime in the future
2886          */
2887         iavf_free_all_rx_resources(adapter);
2888         iavf_free_all_tx_resources(adapter);
2889
2890         adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
2891         /* kill and reinit the admin queue */
2892         iavf_shutdown_adminq(hw);
2893         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2894         status = iavf_init_adminq(hw);
2895         if (status) {
2896                 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
2897                          status);
2898                 goto reset_err;
2899         }
2900         adapter->aq_required = 0;
2901
2902         if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
2903             (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
2904                 err = iavf_reinit_interrupt_scheme(adapter);
2905                 if (err)
2906                         goto reset_err;
2907         }
2908
2909         if (RSS_AQ(adapter)) {
2910                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2911         } else {
2912                 err = iavf_init_rss(adapter);
2913                 if (err)
2914                         goto reset_err;
2915         }
2916
2917         adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
2918         /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
2919          * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
2920          * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
2921          * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
2922          * been successfully sent and negotiated
2923          */
2924         adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
2925         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
2926
2927         spin_lock_bh(&adapter->mac_vlan_list_lock);
2928
2929         /* Delete filter for the current MAC address, it could have
2930          * been changed by the PF via administratively set MAC.
2931          * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
2932          */
2933         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2934                 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
2935                         list_del(&f->list);
2936                         kfree(f);
2937                 }
2938         }
2939         /* re-add all MAC filters */
2940         list_for_each_entry(f, &adapter->mac_filter_list, list) {
2941                 f->add = true;
2942         }
2943         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2944
2945         /* check if TCs are running and re-add all cloud filters */
2946         spin_lock_bh(&adapter->cloud_filter_list_lock);
2947         if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
2948             adapter->num_tc) {
2949                 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
2950                         cf->add = true;
2951                 }
2952         }
2953         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2954
2955         adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
2956         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
2957         iavf_misc_irq_enable(adapter);
2958
2959         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2);
2960
2961         /* We were running when the reset started, so we need to restore some
2962          * state here.
2963          */
2964         if (running) {
2965                 /* allocate transmit descriptors */
2966                 err = iavf_setup_all_tx_resources(adapter);
2967                 if (err)
2968                         goto reset_err;
2969
2970                 /* allocate receive descriptors */
2971                 err = iavf_setup_all_rx_resources(adapter);
2972                 if (err)
2973                         goto reset_err;
2974
2975                 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
2976                     (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
2977                         err = iavf_request_traffic_irqs(adapter, netdev->name);
2978                         if (err)
2979                                 goto reset_err;
2980
2981                         adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
2982                 }
2983
2984                 iavf_configure(adapter);
2985
2986                 /* iavf_up_complete() will switch device back
2987                  * to __IAVF_RUNNING
2988                  */
2989                 iavf_up_complete(adapter);
2990
2991                 iavf_irq_enable(adapter, true);
2992         } else {
2993                 iavf_change_state(adapter, __IAVF_DOWN);
2994                 wake_up(&adapter->down_waitqueue);
2995         }
2996
2997         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
2998
2999         mutex_unlock(&adapter->client_lock);
3000         mutex_unlock(&adapter->crit_lock);
3001
3002         return;
3003 reset_err:
3004         mutex_unlock(&adapter->client_lock);
3005         mutex_unlock(&adapter->crit_lock);
3006         if (running)
3007                 iavf_change_state(adapter, __IAVF_RUNNING);
3008         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3009         iavf_close(netdev);
3010 }
3011
3012 /**
3013  * iavf_adminq_task - worker thread to clean the admin queue
3014  * @work: pointer to work_struct containing our data
3015  **/
3016 static void iavf_adminq_task(struct work_struct *work)
3017 {
3018         struct iavf_adapter *adapter =
3019                 container_of(work, struct iavf_adapter, adminq_task);
3020         struct iavf_hw *hw = &adapter->hw;
3021         struct iavf_arq_event_info event;
3022         enum virtchnl_ops v_op;
3023         enum iavf_status ret, v_ret;
3024         u32 val, oldval;
3025         u16 pending;
3026
3027         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3028                 goto out;
3029
3030         if (!mutex_trylock(&adapter->crit_lock)) {
3031                 if (adapter->state == __IAVF_REMOVE)
3032                         return;
3033
3034                 queue_work(iavf_wq, &adapter->adminq_task);
3035                 goto out;
3036         }
3037
3038         event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3039         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3040         if (!event.msg_buf)
3041                 goto out;
3042
3043         do {
3044                 ret = iavf_clean_arq_element(hw, &event, &pending);
3045                 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3046                 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3047
3048                 if (ret || !v_op)
3049                         break; /* No event to process or error cleaning ARQ */
3050
3051                 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
3052                                          event.msg_len);
3053                 if (pending != 0)
3054                         memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3055         } while (pending);
3056         mutex_unlock(&adapter->crit_lock);
3057
3058         if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES)) {
3059                 if (adapter->netdev_registered ||
3060                     !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
3061                         struct net_device *netdev = adapter->netdev;
3062
3063                         rtnl_lock();
3064                         netdev_update_features(netdev);
3065                         rtnl_unlock();
3066                         /* Request VLAN offload settings */
3067                         if (VLAN_V2_ALLOWED(adapter))
3068                                 iavf_set_vlan_offload_features
3069                                         (adapter, 0, netdev->features);
3070
3071                         iavf_set_queue_vlan_tag_loc(adapter);
3072                 }
3073
3074                 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
3075         }
3076         if ((adapter->flags &
3077              (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
3078             adapter->state == __IAVF_RESETTING)
3079                 goto freedom;
3080
3081         /* check for error indications */
3082         val = rd32(hw, hw->aq.arq.len);
3083         if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3084                 goto freedom;
3085         oldval = val;
3086         if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3087                 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3088                 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3089         }
3090         if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3091                 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3092                 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3093         }
3094         if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3095                 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3096                 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3097         }
3098         if (oldval != val)
3099                 wr32(hw, hw->aq.arq.len, val);
3100
3101         val = rd32(hw, hw->aq.asq.len);
3102         oldval = val;
3103         if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3104                 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3105                 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3106         }
3107         if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3108                 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3109                 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3110         }
3111         if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3112                 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3113                 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3114         }
3115         if (oldval != val)
3116                 wr32(hw, hw->aq.asq.len, val);
3117
3118 freedom:
3119         kfree(event.msg_buf);
3120 out:
3121         /* re-enable Admin queue interrupt cause */
3122         iavf_misc_irq_enable(adapter);
3123 }
3124
3125 /**
3126  * iavf_client_task - worker thread to perform client work
3127  * @work: pointer to work_struct containing our data
3128  *
3129  * This task handles client interactions. Because client calls can be
3130  * reentrant, we can't handle them in the watchdog.
3131  **/
3132 static void iavf_client_task(struct work_struct *work)
3133 {
3134         struct iavf_adapter *adapter =
3135                 container_of(work, struct iavf_adapter, client_task.work);
3136
3137         /* If we can't get the client bit, just give up. We'll be rescheduled
3138          * later.
3139          */
3140
3141         if (!mutex_trylock(&adapter->client_lock))
3142                 return;
3143
3144         if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
3145                 iavf_client_subtask(adapter);
3146                 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3147                 goto out;
3148         }
3149         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
3150                 iavf_notify_client_l2_params(&adapter->vsi);
3151                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
3152                 goto out;
3153         }
3154         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
3155                 iavf_notify_client_close(&adapter->vsi, false);
3156                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3157                 goto out;
3158         }
3159         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
3160                 iavf_notify_client_open(&adapter->vsi);
3161                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
3162         }
3163 out:
3164         mutex_unlock(&adapter->client_lock);
3165 }
3166
3167 /**
3168  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
3169  * @adapter: board private structure
3170  *
3171  * Free all transmit software resources
3172  **/
3173 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3174 {
3175         int i;
3176
3177         if (!adapter->tx_rings)
3178                 return;
3179
3180         for (i = 0; i < adapter->num_active_queues; i++)
3181                 if (adapter->tx_rings[i].desc)
3182                         iavf_free_tx_resources(&adapter->tx_rings[i]);
3183 }
3184
3185 /**
3186  * iavf_setup_all_tx_resources - allocate all queues Tx resources
3187  * @adapter: board private structure
3188  *
3189  * If this function returns with an error, then it's possible one or
3190  * more of the rings is populated (while the rest are not).  It is the
3191  * callers duty to clean those orphaned rings.
3192  *
3193  * Return 0 on success, negative on failure
3194  **/
3195 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3196 {
3197         int i, err = 0;
3198
3199         for (i = 0; i < adapter->num_active_queues; i++) {
3200                 adapter->tx_rings[i].count = adapter->tx_desc_count;
3201                 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
3202                 if (!err)
3203                         continue;
3204                 dev_err(&adapter->pdev->dev,
3205                         "Allocation for Tx Queue %u failed\n", i);
3206                 break;
3207         }
3208
3209         return err;
3210 }
3211
3212 /**
3213  * iavf_setup_all_rx_resources - allocate all queues Rx resources
3214  * @adapter: board private structure
3215  *
3216  * If this function returns with an error, then it's possible one or
3217  * more of the rings is populated (while the rest are not).  It is the
3218  * callers duty to clean those orphaned rings.
3219  *
3220  * Return 0 on success, negative on failure
3221  **/
3222 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3223 {
3224         int i, err = 0;
3225
3226         for (i = 0; i < adapter->num_active_queues; i++) {
3227                 adapter->rx_rings[i].count = adapter->rx_desc_count;
3228                 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
3229                 if (!err)
3230                         continue;
3231                 dev_err(&adapter->pdev->dev,
3232                         "Allocation for Rx Queue %u failed\n", i);
3233                 break;
3234         }
3235         return err;
3236 }
3237
3238 /**
3239  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
3240  * @adapter: board private structure
3241  *
3242  * Free all receive software resources
3243  **/
3244 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3245 {
3246         int i;
3247
3248         if (!adapter->rx_rings)
3249                 return;
3250
3251         for (i = 0; i < adapter->num_active_queues; i++)
3252                 if (adapter->rx_rings[i].desc)
3253                         iavf_free_rx_resources(&adapter->rx_rings[i]);
3254 }
3255
3256 /**
3257  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3258  * @adapter: board private structure
3259  * @max_tx_rate: max Tx bw for a tc
3260  **/
3261 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3262                                       u64 max_tx_rate)
3263 {
3264         int speed = 0, ret = 0;
3265
3266         if (ADV_LINK_SUPPORT(adapter)) {
3267                 if (adapter->link_speed_mbps < U32_MAX) {
3268                         speed = adapter->link_speed_mbps;
3269                         goto validate_bw;
3270                 } else {
3271                         dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3272                         return -EINVAL;
3273                 }
3274         }
3275
3276         switch (adapter->link_speed) {
3277         case VIRTCHNL_LINK_SPEED_40GB:
3278                 speed = SPEED_40000;
3279                 break;
3280         case VIRTCHNL_LINK_SPEED_25GB:
3281                 speed = SPEED_25000;
3282                 break;
3283         case VIRTCHNL_LINK_SPEED_20GB:
3284                 speed = SPEED_20000;
3285                 break;
3286         case VIRTCHNL_LINK_SPEED_10GB:
3287                 speed = SPEED_10000;
3288                 break;
3289         case VIRTCHNL_LINK_SPEED_5GB:
3290                 speed = SPEED_5000;
3291                 break;
3292         case VIRTCHNL_LINK_SPEED_2_5GB:
3293                 speed = SPEED_2500;
3294                 break;
3295         case VIRTCHNL_LINK_SPEED_1GB:
3296                 speed = SPEED_1000;
3297                 break;
3298         case VIRTCHNL_LINK_SPEED_100MB:
3299                 speed = SPEED_100;
3300                 break;
3301         default:
3302                 break;
3303         }
3304
3305 validate_bw:
3306         if (max_tx_rate > speed) {
3307                 dev_err(&adapter->pdev->dev,
3308                         "Invalid tx rate specified\n");
3309                 ret = -EINVAL;
3310         }
3311
3312         return ret;
3313 }
3314
3315 /**
3316  * iavf_validate_ch_config - validate queue mapping info
3317  * @adapter: board private structure
3318  * @mqprio_qopt: queue parameters
3319  *
3320  * This function validates if the config provided by the user to
3321  * configure queue channels is valid or not. Returns 0 on a valid
3322  * config.
3323  **/
3324 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3325                                    struct tc_mqprio_qopt_offload *mqprio_qopt)
3326 {
3327         u64 total_max_rate = 0;
3328         int i, num_qps = 0;
3329         u64 tx_rate = 0;
3330         int ret = 0;
3331
3332         if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3333             mqprio_qopt->qopt.num_tc < 1)
3334                 return -EINVAL;
3335
3336         for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3337                 if (!mqprio_qopt->qopt.count[i] ||
3338                     mqprio_qopt->qopt.offset[i] != num_qps)
3339                         return -EINVAL;
3340                 if (mqprio_qopt->min_rate[i]) {
3341                         dev_err(&adapter->pdev->dev,
3342                                 "Invalid min tx rate (greater than 0) specified\n");
3343                         return -EINVAL;
3344                 }
3345                 /*convert to Mbps */
3346                 tx_rate = div_u64(mqprio_qopt->max_rate[i],
3347                                   IAVF_MBPS_DIVISOR);
3348                 total_max_rate += tx_rate;
3349                 num_qps += mqprio_qopt->qopt.count[i];
3350         }
3351         if (num_qps > adapter->num_active_queues) {
3352                 dev_err(&adapter->pdev->dev,
3353                         "Cannot support requested number of queues\n");
3354                 return -EINVAL;
3355         }
3356
3357         ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
3358         return ret;
3359 }
3360
3361 /**
3362  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3363  * @adapter: board private structure
3364  **/
3365 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3366 {
3367         struct iavf_cloud_filter *cf, *cftmp;
3368
3369         spin_lock_bh(&adapter->cloud_filter_list_lock);
3370         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3371                                  list) {
3372                 list_del(&cf->list);
3373                 kfree(cf);
3374                 adapter->num_cloud_filters--;
3375         }
3376         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3377 }
3378
3379 /**
3380  * __iavf_setup_tc - configure multiple traffic classes
3381  * @netdev: network interface device structure
3382  * @type_data: tc offload data
3383  *
3384  * This function processes the config information provided by the
3385  * user to configure traffic classes/queue channels and packages the
3386  * information to request the PF to setup traffic classes.
3387  *
3388  * Returns 0 on success.
3389  **/
3390 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3391 {
3392         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3393         struct iavf_adapter *adapter = netdev_priv(netdev);
3394         struct virtchnl_vf_resource *vfres = adapter->vf_res;
3395         u8 num_tc = 0, total_qps = 0;
3396         int ret = 0, netdev_tc = 0;
3397         u64 max_tx_rate;
3398         u16 mode;
3399         int i;
3400
3401         num_tc = mqprio_qopt->qopt.num_tc;
3402         mode = mqprio_qopt->mode;
3403
3404         /* delete queue_channel */
3405         if (!mqprio_qopt->qopt.hw) {
3406                 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3407                         /* reset the tc configuration */
3408                         netdev_reset_tc(netdev);
3409                         adapter->num_tc = 0;
3410                         netif_tx_stop_all_queues(netdev);
3411                         netif_tx_disable(netdev);
3412                         iavf_del_all_cloud_filters(adapter);
3413                         adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3414                         goto exit;
3415                 } else {
3416                         return -EINVAL;
3417                 }
3418         }
3419
3420         /* add queue channel */
3421         if (mode == TC_MQPRIO_MODE_CHANNEL) {
3422                 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3423                         dev_err(&adapter->pdev->dev, "ADq not supported\n");
3424                         return -EOPNOTSUPP;
3425                 }
3426                 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3427                         dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3428                         return -EINVAL;
3429                 }
3430
3431                 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3432                 if (ret)
3433                         return ret;
3434                 /* Return if same TC config is requested */
3435                 if (adapter->num_tc == num_tc)
3436                         return 0;
3437                 adapter->num_tc = num_tc;
3438
3439                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3440                         if (i < num_tc) {
3441                                 adapter->ch_config.ch_info[i].count =
3442                                         mqprio_qopt->qopt.count[i];
3443                                 adapter->ch_config.ch_info[i].offset =
3444                                         mqprio_qopt->qopt.offset[i];
3445                                 total_qps += mqprio_qopt->qopt.count[i];
3446                                 max_tx_rate = mqprio_qopt->max_rate[i];
3447                                 /* convert to Mbps */
3448                                 max_tx_rate = div_u64(max_tx_rate,
3449                                                       IAVF_MBPS_DIVISOR);
3450                                 adapter->ch_config.ch_info[i].max_tx_rate =
3451                                         max_tx_rate;
3452                         } else {
3453                                 adapter->ch_config.ch_info[i].count = 1;
3454                                 adapter->ch_config.ch_info[i].offset = 0;
3455                         }
3456                 }
3457                 adapter->ch_config.total_qps = total_qps;
3458                 netif_tx_stop_all_queues(netdev);
3459                 netif_tx_disable(netdev);
3460                 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3461                 netdev_reset_tc(netdev);
3462                 /* Report the tc mapping up the stack */
3463                 netdev_set_num_tc(adapter->netdev, num_tc);
3464                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3465                         u16 qcount = mqprio_qopt->qopt.count[i];
3466                         u16 qoffset = mqprio_qopt->qopt.offset[i];
3467
3468                         if (i < num_tc)
3469                                 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
3470                                                     qoffset);
3471                 }
3472         }
3473 exit:
3474         return ret;
3475 }
3476
3477 /**
3478  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3479  * @adapter: board private structure
3480  * @f: pointer to struct flow_cls_offload
3481  * @filter: pointer to cloud filter structure
3482  */
3483 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3484                                  struct flow_cls_offload *f,
3485                                  struct iavf_cloud_filter *filter)
3486 {
3487         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
3488         struct flow_dissector *dissector = rule->match.dissector;
3489         u16 n_proto_mask = 0;
3490         u16 n_proto_key = 0;
3491         u8 field_flags = 0;
3492         u16 addr_type = 0;
3493         u16 n_proto = 0;
3494         int i = 0;
3495         struct virtchnl_filter *vf = &filter->f;
3496
3497         if (dissector->used_keys &
3498             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
3499               BIT(FLOW_DISSECTOR_KEY_BASIC) |
3500               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3501               BIT(FLOW_DISSECTOR_KEY_VLAN) |
3502               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3503               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3504               BIT(FLOW_DISSECTOR_KEY_PORTS) |
3505               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3506                 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
3507                         dissector->used_keys);
3508                 return -EOPNOTSUPP;
3509         }
3510
3511         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3512                 struct flow_match_enc_keyid match;
3513
3514                 flow_rule_match_enc_keyid(rule, &match);
3515                 if (match.mask->keyid != 0)
3516                         field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3517         }
3518
3519         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
3520                 struct flow_match_basic match;
3521
3522                 flow_rule_match_basic(rule, &match);
3523                 n_proto_key = ntohs(match.key->n_proto);
3524                 n_proto_mask = ntohs(match.mask->n_proto);
3525
3526                 if (n_proto_key == ETH_P_ALL) {
3527                         n_proto_key = 0;
3528                         n_proto_mask = 0;
3529                 }
3530                 n_proto = n_proto_key & n_proto_mask;
3531                 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3532                         return -EINVAL;
3533                 if (n_proto == ETH_P_IPV6) {
3534                         /* specify flow type as TCP IPv6 */
3535                         vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3536                 }
3537
3538                 if (match.key->ip_proto != IPPROTO_TCP) {
3539                         dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3540                         return -EINVAL;
3541                 }
3542         }
3543
3544         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3545                 struct flow_match_eth_addrs match;
3546
3547                 flow_rule_match_eth_addrs(rule, &match);
3548
3549                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
3550                 if (!is_zero_ether_addr(match.mask->dst)) {
3551                         if (is_broadcast_ether_addr(match.mask->dst)) {
3552                                 field_flags |= IAVF_CLOUD_FIELD_OMAC;
3553                         } else {
3554                                 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3555                                         match.mask->dst);
3556                                 return -EINVAL;
3557                         }
3558                 }
3559
3560                 if (!is_zero_ether_addr(match.mask->src)) {
3561                         if (is_broadcast_ether_addr(match.mask->src)) {
3562                                 field_flags |= IAVF_CLOUD_FIELD_IMAC;
3563                         } else {
3564                                 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3565                                         match.mask->src);
3566                                 return -EINVAL;
3567                         }
3568                 }
3569
3570                 if (!is_zero_ether_addr(match.key->dst))
3571                         if (is_valid_ether_addr(match.key->dst) ||
3572                             is_multicast_ether_addr(match.key->dst)) {
3573                                 /* set the mask if a valid dst_mac address */
3574                                 for (i = 0; i < ETH_ALEN; i++)
3575                                         vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3576                                 ether_addr_copy(vf->data.tcp_spec.dst_mac,
3577                                                 match.key->dst);
3578                         }
3579
3580                 if (!is_zero_ether_addr(match.key->src))
3581                         if (is_valid_ether_addr(match.key->src) ||
3582                             is_multicast_ether_addr(match.key->src)) {
3583                                 /* set the mask if a valid dst_mac address */
3584                                 for (i = 0; i < ETH_ALEN; i++)
3585                                         vf->mask.tcp_spec.src_mac[i] |= 0xff;
3586                                 ether_addr_copy(vf->data.tcp_spec.src_mac,
3587                                                 match.key->src);
3588                 }
3589         }
3590
3591         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
3592                 struct flow_match_vlan match;
3593
3594                 flow_rule_match_vlan(rule, &match);
3595                 if (match.mask->vlan_id) {
3596                         if (match.mask->vlan_id == VLAN_VID_MASK) {
3597                                 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3598                         } else {
3599                                 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3600                                         match.mask->vlan_id);
3601                                 return -EINVAL;
3602                         }
3603                 }
3604                 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3605                 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3606         }
3607
3608         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
3609                 struct flow_match_control match;
3610
3611                 flow_rule_match_control(rule, &match);
3612                 addr_type = match.key->addr_type;
3613         }
3614
3615         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3616                 struct flow_match_ipv4_addrs match;
3617
3618                 flow_rule_match_ipv4_addrs(rule, &match);
3619                 if (match.mask->dst) {
3620                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3621                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3622                         } else {
3623                                 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3624                                         be32_to_cpu(match.mask->dst));
3625                                 return -EINVAL;
3626                         }
3627                 }
3628
3629                 if (match.mask->src) {
3630                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
3631                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3632                         } else {
3633                                 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3634                                         be32_to_cpu(match.mask->dst));
3635                                 return -EINVAL;
3636                         }
3637                 }
3638
3639                 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3640                         dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3641                         return -EINVAL;
3642                 }
3643                 if (match.key->dst) {
3644                         vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3645                         vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3646                 }
3647                 if (match.key->src) {
3648                         vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3649                         vf->data.tcp_spec.src_ip[0] = match.key->src;
3650                 }
3651         }
3652
3653         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3654                 struct flow_match_ipv6_addrs match;
3655
3656                 flow_rule_match_ipv6_addrs(rule, &match);
3657
3658                 /* validate mask, make sure it is not IPV6_ADDR_ANY */
3659                 if (ipv6_addr_any(&match.mask->dst)) {
3660                         dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3661                                 IPV6_ADDR_ANY);
3662                         return -EINVAL;
3663                 }
3664
3665                 /* src and dest IPv6 address should not be LOOPBACK
3666                  * (0:0:0:0:0:0:0:1) which can be represented as ::1
3667                  */
3668                 if (ipv6_addr_loopback(&match.key->dst) ||
3669                     ipv6_addr_loopback(&match.key->src)) {
3670                         dev_err(&adapter->pdev->dev,
3671                                 "ipv6 addr should not be loopback\n");
3672                         return -EINVAL;
3673                 }
3674                 if (!ipv6_addr_any(&match.mask->dst) ||
3675                     !ipv6_addr_any(&match.mask->src))
3676                         field_flags |= IAVF_CLOUD_FIELD_IIP;
3677
3678                 for (i = 0; i < 4; i++)
3679                         vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3680                 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3681                        sizeof(vf->data.tcp_spec.dst_ip));
3682                 for (i = 0; i < 4; i++)
3683                         vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3684                 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3685                        sizeof(vf->data.tcp_spec.src_ip));
3686         }
3687         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
3688                 struct flow_match_ports match;
3689
3690                 flow_rule_match_ports(rule, &match);
3691                 if (match.mask->src) {
3692                         if (match.mask->src == cpu_to_be16(0xffff)) {
3693                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3694                         } else {
3695                                 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3696                                         be16_to_cpu(match.mask->src));
3697                                 return -EINVAL;
3698                         }
3699                 }
3700
3701                 if (match.mask->dst) {
3702                         if (match.mask->dst == cpu_to_be16(0xffff)) {
3703                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3704                         } else {
3705                                 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3706                                         be16_to_cpu(match.mask->dst));
3707                                 return -EINVAL;
3708                         }
3709                 }
3710                 if (match.key->dst) {
3711                         vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3712                         vf->data.tcp_spec.dst_port = match.key->dst;
3713                 }
3714
3715                 if (match.key->src) {
3716                         vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3717                         vf->data.tcp_spec.src_port = match.key->src;
3718                 }
3719         }
3720         vf->field_flags = field_flags;
3721
3722         return 0;
3723 }
3724
3725 /**
3726  * iavf_handle_tclass - Forward to a traffic class on the device
3727  * @adapter: board private structure
3728  * @tc: traffic class index on the device
3729  * @filter: pointer to cloud filter structure
3730  */
3731 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3732                               struct iavf_cloud_filter *filter)
3733 {
3734         if (tc == 0)
3735                 return 0;
3736         if (tc < adapter->num_tc) {
3737                 if (!filter->f.data.tcp_spec.dst_port) {
3738                         dev_err(&adapter->pdev->dev,
3739                                 "Specify destination port to redirect to traffic class other than TC0\n");
3740                         return -EINVAL;
3741                 }
3742         }
3743         /* redirect to a traffic class on the same device */
3744         filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3745         filter->f.action_meta = tc;
3746         return 0;
3747 }
3748
3749 /**
3750  * iavf_configure_clsflower - Add tc flower filters
3751  * @adapter: board private structure
3752  * @cls_flower: Pointer to struct flow_cls_offload
3753  */
3754 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3755                                     struct flow_cls_offload *cls_flower)
3756 {
3757         int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3758         struct iavf_cloud_filter *filter = NULL;
3759         int err = -EINVAL, count = 50;
3760
3761         if (tc < 0) {
3762                 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3763                 return -EINVAL;
3764         }
3765
3766         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
3767         if (!filter)
3768                 return -ENOMEM;
3769
3770         while (!mutex_trylock(&adapter->crit_lock)) {
3771                 if (--count == 0) {
3772                         kfree(filter);
3773                         return err;
3774                 }
3775                 udelay(1);
3776         }
3777
3778         filter->cookie = cls_flower->cookie;
3779
3780         /* set the mask to all zeroes to begin with */
3781         memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
3782         /* start out with flow type and eth type IPv4 to begin with */
3783         filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
3784         err = iavf_parse_cls_flower(adapter, cls_flower, filter);
3785         if (err)
3786                 goto err;
3787
3788         err = iavf_handle_tclass(adapter, tc, filter);
3789         if (err)
3790                 goto err;
3791
3792         /* add filter to the list */
3793         spin_lock_bh(&adapter->cloud_filter_list_lock);
3794         list_add_tail(&filter->list, &adapter->cloud_filter_list);
3795         adapter->num_cloud_filters++;
3796         filter->add = true;
3797         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3798         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3799 err:
3800         if (err)
3801                 kfree(filter);
3802
3803         mutex_unlock(&adapter->crit_lock);
3804         return err;
3805 }
3806
3807 /* iavf_find_cf - Find the cloud filter in the list
3808  * @adapter: Board private structure
3809  * @cookie: filter specific cookie
3810  *
3811  * Returns ptr to the filter object or NULL. Must be called while holding the
3812  * cloud_filter_list_lock.
3813  */
3814 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3815                                               unsigned long *cookie)
3816 {
3817         struct iavf_cloud_filter *filter = NULL;
3818
3819         if (!cookie)
3820                 return NULL;
3821
3822         list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3823                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3824                         return filter;
3825         }
3826         return NULL;
3827 }
3828
3829 /**
3830  * iavf_delete_clsflower - Remove tc flower filters
3831  * @adapter: board private structure
3832  * @cls_flower: Pointer to struct flow_cls_offload
3833  */
3834 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
3835                                  struct flow_cls_offload *cls_flower)
3836 {
3837         struct iavf_cloud_filter *filter = NULL;
3838         int err = 0;
3839
3840         spin_lock_bh(&adapter->cloud_filter_list_lock);
3841         filter = iavf_find_cf(adapter, &cls_flower->cookie);
3842         if (filter) {
3843                 filter->del = true;
3844                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
3845         } else {
3846                 err = -EINVAL;
3847         }
3848         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3849
3850         return err;
3851 }
3852
3853 /**
3854  * iavf_setup_tc_cls_flower - flower classifier offloads
3855  * @adapter: board private structure
3856  * @cls_flower: pointer to flow_cls_offload struct with flow info
3857  */
3858 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
3859                                     struct flow_cls_offload *cls_flower)
3860 {
3861         switch (cls_flower->command) {
3862         case FLOW_CLS_REPLACE:
3863                 return iavf_configure_clsflower(adapter, cls_flower);
3864         case FLOW_CLS_DESTROY:
3865                 return iavf_delete_clsflower(adapter, cls_flower);
3866         case FLOW_CLS_STATS:
3867                 return -EOPNOTSUPP;
3868         default:
3869                 return -EOPNOTSUPP;
3870         }
3871 }
3872
3873 /**
3874  * iavf_setup_tc_block_cb - block callback for tc
3875  * @type: type of offload
3876  * @type_data: offload data
3877  * @cb_priv:
3878  *
3879  * This function is the block callback for traffic classes
3880  **/
3881 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3882                                   void *cb_priv)
3883 {
3884         struct iavf_adapter *adapter = cb_priv;
3885
3886         if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
3887                 return -EOPNOTSUPP;
3888
3889         switch (type) {
3890         case TC_SETUP_CLSFLOWER:
3891                 return iavf_setup_tc_cls_flower(cb_priv, type_data);
3892         default:
3893                 return -EOPNOTSUPP;
3894         }
3895 }
3896
3897 static LIST_HEAD(iavf_block_cb_list);
3898
3899 /**
3900  * iavf_setup_tc - configure multiple traffic classes
3901  * @netdev: network interface device structure
3902  * @type: type of offload
3903  * @type_data: tc offload data
3904  *
3905  * This function is the callback to ndo_setup_tc in the
3906  * netdev_ops.
3907  *
3908  * Returns 0 on success
3909  **/
3910 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
3911                          void *type_data)
3912 {
3913         struct iavf_adapter *adapter = netdev_priv(netdev);
3914
3915         switch (type) {
3916         case TC_SETUP_QDISC_MQPRIO:
3917                 return __iavf_setup_tc(netdev, type_data);
3918         case TC_SETUP_BLOCK:
3919                 return flow_block_cb_setup_simple(type_data,
3920                                                   &iavf_block_cb_list,
3921                                                   iavf_setup_tc_block_cb,
3922                                                   adapter, adapter, true);
3923         default:
3924                 return -EOPNOTSUPP;
3925         }
3926 }
3927
3928 /**
3929  * iavf_open - Called when a network interface is made active
3930  * @netdev: network interface device structure
3931  *
3932  * Returns 0 on success, negative value on failure
3933  *
3934  * The open entry point is called when a network interface is made
3935  * active by the system (IFF_UP).  At this point all resources needed
3936  * for transmit and receive operations are allocated, the interrupt
3937  * handler is registered with the OS, the watchdog is started,
3938  * and the stack is notified that the interface is ready.
3939  **/
3940 static int iavf_open(struct net_device *netdev)
3941 {
3942         struct iavf_adapter *adapter = netdev_priv(netdev);
3943         int err;
3944
3945         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
3946                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
3947                 return -EIO;
3948         }
3949
3950         while (!mutex_trylock(&adapter->crit_lock))
3951                 usleep_range(500, 1000);
3952
3953         if (adapter->state != __IAVF_DOWN) {
3954                 err = -EBUSY;
3955                 goto err_unlock;
3956         }
3957
3958         if (adapter->state == __IAVF_RUNNING &&
3959             !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
3960                 dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
3961                 err = 0;
3962                 goto err_unlock;
3963         }
3964
3965         /* allocate transmit descriptors */
3966         err = iavf_setup_all_tx_resources(adapter);
3967         if (err)
3968                 goto err_setup_tx;
3969
3970         /* allocate receive descriptors */
3971         err = iavf_setup_all_rx_resources(adapter);
3972         if (err)
3973                 goto err_setup_rx;
3974
3975         /* clear any pending interrupts, may auto mask */
3976         err = iavf_request_traffic_irqs(adapter, netdev->name);
3977         if (err)
3978                 goto err_req_irq;
3979
3980         spin_lock_bh(&adapter->mac_vlan_list_lock);
3981
3982         iavf_add_filter(adapter, adapter->hw.mac.addr);
3983
3984         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3985
3986         /* Restore VLAN filters that were removed with IFF_DOWN */
3987         iavf_restore_filters(adapter);
3988
3989         iavf_configure(adapter);
3990
3991         iavf_up_complete(adapter);
3992
3993         iavf_irq_enable(adapter, true);
3994
3995         mutex_unlock(&adapter->crit_lock);
3996
3997         return 0;
3998
3999 err_req_irq:
4000         iavf_down(adapter);
4001         iavf_free_traffic_irqs(adapter);
4002 err_setup_rx:
4003         iavf_free_all_rx_resources(adapter);
4004 err_setup_tx:
4005         iavf_free_all_tx_resources(adapter);
4006 err_unlock:
4007         mutex_unlock(&adapter->crit_lock);
4008
4009         return err;
4010 }
4011
4012 /**
4013  * iavf_close - Disables a network interface
4014  * @netdev: network interface device structure
4015  *
4016  * Returns 0, this is not allowed to fail
4017  *
4018  * The close entry point is called when an interface is de-activated
4019  * by the OS.  The hardware is still under the drivers control, but
4020  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4021  * are freed, along with all transmit and receive resources.
4022  **/
4023 static int iavf_close(struct net_device *netdev)
4024 {
4025         struct iavf_adapter *adapter = netdev_priv(netdev);
4026         int status;
4027
4028         mutex_lock(&adapter->crit_lock);
4029
4030         if (adapter->state <= __IAVF_DOWN_PENDING) {
4031                 mutex_unlock(&adapter->crit_lock);
4032                 return 0;
4033         }
4034
4035         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
4036         if (CLIENT_ENABLED(adapter))
4037                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
4038
4039         iavf_down(adapter);
4040         iavf_change_state(adapter, __IAVF_DOWN_PENDING);
4041         iavf_free_traffic_irqs(adapter);
4042
4043         mutex_unlock(&adapter->crit_lock);
4044
4045         /* We explicitly don't free resources here because the hardware is
4046          * still active and can DMA into memory. Resources are cleared in
4047          * iavf_virtchnl_completion() after we get confirmation from the PF
4048          * driver that the rings have been stopped.
4049          *
4050          * Also, we wait for state to transition to __IAVF_DOWN before
4051          * returning. State change occurs in iavf_virtchnl_completion() after
4052          * VF resources are released (which occurs after PF driver processes and
4053          * responds to admin queue commands).
4054          */
4055
4056         status = wait_event_timeout(adapter->down_waitqueue,
4057                                     adapter->state == __IAVF_DOWN,
4058                                     msecs_to_jiffies(500));
4059         if (!status)
4060                 netdev_warn(netdev, "Device resources not yet released\n");
4061         return 0;
4062 }
4063
4064 /**
4065  * iavf_change_mtu - Change the Maximum Transfer Unit
4066  * @netdev: network interface device structure
4067  * @new_mtu: new value for maximum frame size
4068  *
4069  * Returns 0 on success, negative on failure
4070  **/
4071 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4072 {
4073         struct iavf_adapter *adapter = netdev_priv(netdev);
4074
4075         netdev_dbg(netdev, "changing MTU from %d to %d\n",
4076                    netdev->mtu, new_mtu);
4077         netdev->mtu = new_mtu;
4078         if (CLIENT_ENABLED(adapter)) {
4079                 iavf_notify_client_l2_params(&adapter->vsi);
4080                 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
4081         }
4082
4083         if (netif_running(netdev)) {
4084                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
4085                 queue_work(iavf_wq, &adapter->reset_task);
4086         }
4087
4088         return 0;
4089 }
4090
4091 #define NETIF_VLAN_OFFLOAD_FEATURES     (NETIF_F_HW_VLAN_CTAG_RX | \
4092                                          NETIF_F_HW_VLAN_CTAG_TX | \
4093                                          NETIF_F_HW_VLAN_STAG_RX | \
4094                                          NETIF_F_HW_VLAN_STAG_TX)
4095
4096 /**
4097  * iavf_set_features - set the netdev feature flags
4098  * @netdev: ptr to the netdev being adjusted
4099  * @features: the feature set that the stack is suggesting
4100  * Note: expects to be called while under rtnl_lock()
4101  **/
4102 static int iavf_set_features(struct net_device *netdev,
4103                              netdev_features_t features)
4104 {
4105         struct iavf_adapter *adapter = netdev_priv(netdev);
4106
4107         /* trigger update on any VLAN feature change */
4108         if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4109             (features & NETIF_VLAN_OFFLOAD_FEATURES))
4110                 iavf_set_vlan_offload_features(adapter, netdev->features,
4111                                                features);
4112
4113         return 0;
4114 }
4115
4116 /**
4117  * iavf_features_check - Validate encapsulated packet conforms to limits
4118  * @skb: skb buff
4119  * @dev: This physical port's netdev
4120  * @features: Offload features that the stack believes apply
4121  **/
4122 static netdev_features_t iavf_features_check(struct sk_buff *skb,
4123                                              struct net_device *dev,
4124                                              netdev_features_t features)
4125 {
4126         size_t len;
4127
4128         /* No point in doing any of this if neither checksum nor GSO are
4129          * being requested for this frame.  We can rule out both by just
4130          * checking for CHECKSUM_PARTIAL
4131          */
4132         if (skb->ip_summed != CHECKSUM_PARTIAL)
4133                 return features;
4134
4135         /* We cannot support GSO if the MSS is going to be less than
4136          * 64 bytes.  If it is then we need to drop support for GSO.
4137          */
4138         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4139                 features &= ~NETIF_F_GSO_MASK;
4140
4141         /* MACLEN can support at most 63 words */
4142         len = skb_network_header(skb) - skb->data;
4143         if (len & ~(63 * 2))
4144                 goto out_err;
4145
4146         /* IPLEN and EIPLEN can support at most 127 dwords */
4147         len = skb_transport_header(skb) - skb_network_header(skb);
4148         if (len & ~(127 * 4))
4149                 goto out_err;
4150
4151         if (skb->encapsulation) {
4152                 /* L4TUNLEN can support 127 words */
4153                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
4154                 if (len & ~(127 * 2))
4155                         goto out_err;
4156
4157                 /* IPLEN can support at most 127 dwords */
4158                 len = skb_inner_transport_header(skb) -
4159                       skb_inner_network_header(skb);
4160                 if (len & ~(127 * 4))
4161                         goto out_err;
4162         }
4163
4164         /* No need to validate L4LEN as TCP is the only protocol with a
4165          * a flexible value and we support all possible values supported
4166          * by TCP, which is at most 15 dwords
4167          */
4168
4169         return features;
4170 out_err:
4171         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4172 }
4173
4174 /**
4175  * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4176  * @adapter: board private structure
4177  *
4178  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4179  * were negotiated determine the VLAN features that can be toggled on and off.
4180  **/
4181 static netdev_features_t
4182 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4183 {
4184         netdev_features_t hw_features = 0;
4185
4186         if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4187                 return hw_features;
4188
4189         /* Enable VLAN features if supported */
4190         if (VLAN_ALLOWED(adapter)) {
4191                 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4192                                 NETIF_F_HW_VLAN_CTAG_RX);
4193         } else if (VLAN_V2_ALLOWED(adapter)) {
4194                 struct virtchnl_vlan_caps *vlan_v2_caps =
4195                         &adapter->vlan_v2_caps;
4196                 struct virtchnl_vlan_supported_caps *stripping_support =
4197                         &vlan_v2_caps->offloads.stripping_support;
4198                 struct virtchnl_vlan_supported_caps *insertion_support =
4199                         &vlan_v2_caps->offloads.insertion_support;
4200
4201                 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4202                     stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4203                         if (stripping_support->outer &
4204                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4205                                 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4206                         if (stripping_support->outer &
4207                             VIRTCHNL_VLAN_ETHERTYPE_88A8)
4208                                 hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4209                 } else if (stripping_support->inner !=
4210                            VIRTCHNL_VLAN_UNSUPPORTED &&
4211                            stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4212                         if (stripping_support->inner &
4213                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4214                                 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4215                 }
4216
4217                 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4218                     insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4219                         if (insertion_support->outer &
4220                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4221                                 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4222                         if (insertion_support->outer &
4223                             VIRTCHNL_VLAN_ETHERTYPE_88A8)
4224                                 hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4225                 } else if (insertion_support->inner &&
4226                            insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4227                         if (insertion_support->inner &
4228                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4229                                 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4230                 }
4231         }
4232
4233         return hw_features;
4234 }
4235
4236 /**
4237  * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4238  * @adapter: board private structure
4239  *
4240  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4241  * were negotiated determine the VLAN features that are enabled by default.
4242  **/
4243 static netdev_features_t
4244 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4245 {
4246         netdev_features_t features = 0;
4247
4248         if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4249                 return features;
4250
4251         if (VLAN_ALLOWED(adapter)) {
4252                 features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4253                         NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4254         } else if (VLAN_V2_ALLOWED(adapter)) {
4255                 struct virtchnl_vlan_caps *vlan_v2_caps =
4256                         &adapter->vlan_v2_caps;
4257                 struct virtchnl_vlan_supported_caps *filtering_support =
4258                         &vlan_v2_caps->filtering.filtering_support;
4259                 struct virtchnl_vlan_supported_caps *stripping_support =
4260                         &vlan_v2_caps->offloads.stripping_support;
4261                 struct virtchnl_vlan_supported_caps *insertion_support =
4262                         &vlan_v2_caps->offloads.insertion_support;
4263                 u32 ethertype_init;
4264
4265                 /* give priority to outer stripping and don't support both outer
4266                  * and inner stripping
4267                  */
4268                 ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4269                 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4270                         if (stripping_support->outer &
4271                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4272                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4273                                 features |= NETIF_F_HW_VLAN_CTAG_RX;
4274                         else if (stripping_support->outer &
4275                                  VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4276                                  ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4277                                 features |= NETIF_F_HW_VLAN_STAG_RX;
4278                 } else if (stripping_support->inner !=
4279                            VIRTCHNL_VLAN_UNSUPPORTED) {
4280                         if (stripping_support->inner &
4281                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4282                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4283                                 features |= NETIF_F_HW_VLAN_CTAG_RX;
4284                 }
4285
4286                 /* give priority to outer insertion and don't support both outer
4287                  * and inner insertion
4288                  */
4289                 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4290                         if (insertion_support->outer &
4291                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4292                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4293                                 features |= NETIF_F_HW_VLAN_CTAG_TX;
4294                         else if (insertion_support->outer &
4295                                  VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4296                                  ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4297                                 features |= NETIF_F_HW_VLAN_STAG_TX;
4298                 } else if (insertion_support->inner !=
4299                            VIRTCHNL_VLAN_UNSUPPORTED) {
4300                         if (insertion_support->inner &
4301                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4302                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4303                                 features |= NETIF_F_HW_VLAN_CTAG_TX;
4304                 }
4305
4306                 /* give priority to outer filtering and don't bother if both
4307                  * outer and inner filtering are enabled
4308                  */
4309                 ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4310                 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4311                         if (filtering_support->outer &
4312                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4313                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4314                                 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4315                         if (filtering_support->outer &
4316                             VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4317                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4318                                 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4319                 } else if (filtering_support->inner !=
4320                            VIRTCHNL_VLAN_UNSUPPORTED) {
4321                         if (filtering_support->inner &
4322                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4323                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4324                                 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4325                         if (filtering_support->inner &
4326                             VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4327                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4328                                 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4329                 }
4330         }
4331
4332         return features;
4333 }
4334
4335 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4336         (!(((requested) & (feature_bit)) && \
4337            !((allowed) & (feature_bit))))
4338
4339 /**
4340  * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4341  * @adapter: board private structure
4342  * @requested_features: stack requested NETDEV features
4343  **/
4344 static netdev_features_t
4345 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4346                               netdev_features_t requested_features)
4347 {
4348         netdev_features_t allowed_features;
4349
4350         allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4351                 iavf_get_netdev_vlan_features(adapter);
4352
4353         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4354                                               allowed_features,
4355                                               NETIF_F_HW_VLAN_CTAG_TX))
4356                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4357
4358         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4359                                               allowed_features,
4360                                               NETIF_F_HW_VLAN_CTAG_RX))
4361                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4362
4363         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4364                                               allowed_features,
4365                                               NETIF_F_HW_VLAN_STAG_TX))
4366                 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4367         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4368                                               allowed_features,
4369                                               NETIF_F_HW_VLAN_STAG_RX))
4370                 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4371
4372         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4373                                               allowed_features,
4374                                               NETIF_F_HW_VLAN_CTAG_FILTER))
4375                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4376
4377         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4378                                               allowed_features,
4379                                               NETIF_F_HW_VLAN_STAG_FILTER))
4380                 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4381
4382         if ((requested_features &
4383              (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4384             (requested_features &
4385              (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4386             adapter->vlan_v2_caps.offloads.ethertype_match ==
4387             VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4388                 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4389                 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4390                                         NETIF_F_HW_VLAN_STAG_TX);
4391         }
4392
4393         return requested_features;
4394 }
4395
4396 /**
4397  * iavf_fix_features - fix up the netdev feature bits
4398  * @netdev: our net device
4399  * @features: desired feature bits
4400  *
4401  * Returns fixed-up features bits
4402  **/
4403 static netdev_features_t iavf_fix_features(struct net_device *netdev,
4404                                            netdev_features_t features)
4405 {
4406         struct iavf_adapter *adapter = netdev_priv(netdev);
4407
4408         return iavf_fix_netdev_vlan_features(adapter, features);
4409 }
4410
4411 static const struct net_device_ops iavf_netdev_ops = {
4412         .ndo_open               = iavf_open,
4413         .ndo_stop               = iavf_close,
4414         .ndo_start_xmit         = iavf_xmit_frame,
4415         .ndo_set_rx_mode        = iavf_set_rx_mode,
4416         .ndo_validate_addr      = eth_validate_addr,
4417         .ndo_set_mac_address    = iavf_set_mac,
4418         .ndo_change_mtu         = iavf_change_mtu,
4419         .ndo_tx_timeout         = iavf_tx_timeout,
4420         .ndo_vlan_rx_add_vid    = iavf_vlan_rx_add_vid,
4421         .ndo_vlan_rx_kill_vid   = iavf_vlan_rx_kill_vid,
4422         .ndo_features_check     = iavf_features_check,
4423         .ndo_fix_features       = iavf_fix_features,
4424         .ndo_set_features       = iavf_set_features,
4425         .ndo_setup_tc           = iavf_setup_tc,
4426 };
4427
4428 /**
4429  * iavf_check_reset_complete - check that VF reset is complete
4430  * @hw: pointer to hw struct
4431  *
4432  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
4433  **/
4434 static int iavf_check_reset_complete(struct iavf_hw *hw)
4435 {
4436         u32 rstat;
4437         int i;
4438
4439         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
4440                 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
4441                              IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
4442                 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
4443                     (rstat == VIRTCHNL_VFR_COMPLETED))
4444                         return 0;
4445                 usleep_range(10, 20);
4446         }
4447         return -EBUSY;
4448 }
4449
4450 /**
4451  * iavf_process_config - Process the config information we got from the PF
4452  * @adapter: board private structure
4453  *
4454  * Verify that we have a valid config struct, and set up our netdev features
4455  * and our VSI struct.
4456  **/
4457 int iavf_process_config(struct iavf_adapter *adapter)
4458 {
4459         struct virtchnl_vf_resource *vfres = adapter->vf_res;
4460         netdev_features_t hw_vlan_features, vlan_features;
4461         struct net_device *netdev = adapter->netdev;
4462         netdev_features_t hw_enc_features;
4463         netdev_features_t hw_features;
4464
4465         hw_enc_features = NETIF_F_SG                    |
4466                           NETIF_F_IP_CSUM               |
4467                           NETIF_F_IPV6_CSUM             |
4468                           NETIF_F_HIGHDMA               |
4469                           NETIF_F_SOFT_FEATURES |
4470                           NETIF_F_TSO                   |
4471                           NETIF_F_TSO_ECN               |
4472                           NETIF_F_TSO6                  |
4473                           NETIF_F_SCTP_CRC              |
4474                           NETIF_F_RXHASH                |
4475                           NETIF_F_RXCSUM                |
4476                           0;
4477
4478         /* advertise to stack only if offloads for encapsulated packets is
4479          * supported
4480          */
4481         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
4482                 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL       |
4483                                    NETIF_F_GSO_GRE              |
4484                                    NETIF_F_GSO_GRE_CSUM         |
4485                                    NETIF_F_GSO_IPXIP4           |
4486                                    NETIF_F_GSO_IPXIP6           |
4487                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
4488                                    NETIF_F_GSO_PARTIAL          |
4489                                    0;
4490
4491                 if (!(vfres->vf_cap_flags &
4492                       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
4493                         netdev->gso_partial_features |=
4494                                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
4495
4496                 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
4497                 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
4498                 netdev->hw_enc_features |= hw_enc_features;
4499         }
4500         /* record features VLANs can make use of */
4501         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
4502
4503         /* Write features and hw_features separately to avoid polluting
4504          * with, or dropping, features that are set when we registered.
4505          */
4506         hw_features = hw_enc_features;
4507
4508         /* get HW VLAN features that can be toggled */
4509         hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
4510
4511         /* Enable cloud filter if ADQ is supported */
4512         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
4513                 hw_features |= NETIF_F_HW_TC;
4514         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
4515                 hw_features |= NETIF_F_GSO_UDP_L4;
4516
4517         netdev->hw_features |= hw_features | hw_vlan_features;
4518         vlan_features = iavf_get_netdev_vlan_features(adapter);
4519
4520         netdev->features |= hw_features | vlan_features;
4521
4522         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
4523                 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4524
4525         netdev->priv_flags |= IFF_UNICAST_FLT;
4526
4527         /* Do not turn on offloads when they are requested to be turned off.
4528          * TSO needs minimum 576 bytes to work correctly.
4529          */
4530         if (netdev->wanted_features) {
4531                 if (!(netdev->wanted_features & NETIF_F_TSO) ||
4532                     netdev->mtu < 576)
4533                         netdev->features &= ~NETIF_F_TSO;
4534                 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
4535                     netdev->mtu < 576)
4536                         netdev->features &= ~NETIF_F_TSO6;
4537                 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
4538                         netdev->features &= ~NETIF_F_TSO_ECN;
4539                 if (!(netdev->wanted_features & NETIF_F_GRO))
4540                         netdev->features &= ~NETIF_F_GRO;
4541                 if (!(netdev->wanted_features & NETIF_F_GSO))
4542                         netdev->features &= ~NETIF_F_GSO;
4543         }
4544
4545         return 0;
4546 }
4547
4548 /**
4549  * iavf_shutdown - Shutdown the device in preparation for a reboot
4550  * @pdev: pci device structure
4551  **/
4552 static void iavf_shutdown(struct pci_dev *pdev)
4553 {
4554         struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
4555         struct net_device *netdev = adapter->netdev;
4556
4557         netif_device_detach(netdev);
4558
4559         if (netif_running(netdev))
4560                 iavf_close(netdev);
4561
4562         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
4563                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
4564         /* Prevent the watchdog from running. */
4565         iavf_change_state(adapter, __IAVF_REMOVE);
4566         adapter->aq_required = 0;
4567         mutex_unlock(&adapter->crit_lock);
4568
4569 #ifdef CONFIG_PM
4570         pci_save_state(pdev);
4571
4572 #endif
4573         pci_disable_device(pdev);
4574 }
4575
4576 /**
4577  * iavf_probe - Device Initialization Routine
4578  * @pdev: PCI device information struct
4579  * @ent: entry in iavf_pci_tbl
4580  *
4581  * Returns 0 on success, negative on failure
4582  *
4583  * iavf_probe initializes an adapter identified by a pci_dev structure.
4584  * The OS initialization, configuring of the adapter private structure,
4585  * and a hardware reset occur.
4586  **/
4587 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4588 {
4589         struct net_device *netdev;
4590         struct iavf_adapter *adapter = NULL;
4591         struct iavf_hw *hw = NULL;
4592         int err;
4593
4594         err = pci_enable_device(pdev);
4595         if (err)
4596                 return err;
4597
4598         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4599         if (err) {
4600                 dev_err(&pdev->dev,
4601                         "DMA configuration failed: 0x%x\n", err);
4602                 goto err_dma;
4603         }
4604
4605         err = pci_request_regions(pdev, iavf_driver_name);
4606         if (err) {
4607                 dev_err(&pdev->dev,
4608                         "pci_request_regions failed 0x%x\n", err);
4609                 goto err_pci_reg;
4610         }
4611
4612         pci_enable_pcie_error_reporting(pdev);
4613
4614         pci_set_master(pdev);
4615
4616         netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
4617                                    IAVF_MAX_REQ_QUEUES);
4618         if (!netdev) {
4619                 err = -ENOMEM;
4620                 goto err_alloc_etherdev;
4621         }
4622
4623         SET_NETDEV_DEV(netdev, &pdev->dev);
4624
4625         pci_set_drvdata(pdev, netdev);
4626         adapter = netdev_priv(netdev);
4627
4628         adapter->netdev = netdev;
4629         adapter->pdev = pdev;
4630
4631         hw = &adapter->hw;
4632         hw->back = adapter;
4633
4634         adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
4635         iavf_change_state(adapter, __IAVF_STARTUP);
4636
4637         /* Call save state here because it relies on the adapter struct. */
4638         pci_save_state(pdev);
4639
4640         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4641                               pci_resource_len(pdev, 0));
4642         if (!hw->hw_addr) {
4643                 err = -EIO;
4644                 goto err_ioremap;
4645         }
4646         hw->vendor_id = pdev->vendor;
4647         hw->device_id = pdev->device;
4648         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
4649         hw->subsystem_vendor_id = pdev->subsystem_vendor;
4650         hw->subsystem_device_id = pdev->subsystem_device;
4651         hw->bus.device = PCI_SLOT(pdev->devfn);
4652         hw->bus.func = PCI_FUNC(pdev->devfn);
4653         hw->bus.bus_id = pdev->bus->number;
4654
4655         /* set up the locks for the AQ, do this only once in probe
4656          * and destroy them only once in remove
4657          */
4658         mutex_init(&adapter->crit_lock);
4659         mutex_init(&adapter->client_lock);
4660         mutex_init(&hw->aq.asq_mutex);
4661         mutex_init(&hw->aq.arq_mutex);
4662
4663         spin_lock_init(&adapter->mac_vlan_list_lock);
4664         spin_lock_init(&adapter->cloud_filter_list_lock);
4665         spin_lock_init(&adapter->fdir_fltr_lock);
4666         spin_lock_init(&adapter->adv_rss_lock);
4667
4668         INIT_LIST_HEAD(&adapter->mac_filter_list);
4669         INIT_LIST_HEAD(&adapter->vlan_filter_list);
4670         INIT_LIST_HEAD(&adapter->cloud_filter_list);
4671         INIT_LIST_HEAD(&adapter->fdir_list_head);
4672         INIT_LIST_HEAD(&adapter->adv_rss_list_head);
4673
4674         INIT_WORK(&adapter->reset_task, iavf_reset_task);
4675         INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
4676         INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
4677         INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
4678         queue_delayed_work(iavf_wq, &adapter->watchdog_task,
4679                            msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
4680
4681         /* Setup the wait queue for indicating transition to down status */
4682         init_waitqueue_head(&adapter->down_waitqueue);
4683
4684         return 0;
4685
4686 err_ioremap:
4687         free_netdev(netdev);
4688 err_alloc_etherdev:
4689         pci_disable_pcie_error_reporting(pdev);
4690         pci_release_regions(pdev);
4691 err_pci_reg:
4692 err_dma:
4693         pci_disable_device(pdev);
4694         return err;
4695 }
4696
4697 /**
4698  * iavf_suspend - Power management suspend routine
4699  * @dev_d: device info pointer
4700  *
4701  * Called when the system (VM) is entering sleep/suspend.
4702  **/
4703 static int __maybe_unused iavf_suspend(struct device *dev_d)
4704 {
4705         struct net_device *netdev = dev_get_drvdata(dev_d);
4706         struct iavf_adapter *adapter = netdev_priv(netdev);
4707
4708         netif_device_detach(netdev);
4709
4710         while (!mutex_trylock(&adapter->crit_lock))
4711                 usleep_range(500, 1000);
4712
4713         if (netif_running(netdev)) {
4714                 rtnl_lock();
4715                 iavf_down(adapter);
4716                 rtnl_unlock();
4717         }
4718         iavf_free_misc_irq(adapter);
4719         iavf_reset_interrupt_capability(adapter);
4720
4721         mutex_unlock(&adapter->crit_lock);
4722
4723         return 0;
4724 }
4725
4726 /**
4727  * iavf_resume - Power management resume routine
4728  * @dev_d: device info pointer
4729  *
4730  * Called when the system (VM) is resumed from sleep/suspend.
4731  **/
4732 static int __maybe_unused iavf_resume(struct device *dev_d)
4733 {
4734         struct pci_dev *pdev = to_pci_dev(dev_d);
4735         struct iavf_adapter *adapter;
4736         u32 err;
4737
4738         adapter = iavf_pdev_to_adapter(pdev);
4739
4740         pci_set_master(pdev);
4741
4742         rtnl_lock();
4743         err = iavf_set_interrupt_capability(adapter);
4744         if (err) {
4745                 rtnl_unlock();
4746                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
4747                 return err;
4748         }
4749         err = iavf_request_misc_irq(adapter);
4750         rtnl_unlock();
4751         if (err) {
4752                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
4753                 return err;
4754         }
4755
4756         queue_work(iavf_wq, &adapter->reset_task);
4757
4758         netif_device_attach(adapter->netdev);
4759
4760         return err;
4761 }
4762
4763 /**
4764  * iavf_remove - Device Removal Routine
4765  * @pdev: PCI device information struct
4766  *
4767  * iavf_remove is called by the PCI subsystem to alert the driver
4768  * that it should release a PCI device.  The could be caused by a
4769  * Hot-Plug event, or because the driver is going to be removed from
4770  * memory.
4771  **/
4772 static void iavf_remove(struct pci_dev *pdev)
4773 {
4774         struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
4775         struct net_device *netdev = adapter->netdev;
4776         struct iavf_fdir_fltr *fdir, *fdirtmp;
4777         struct iavf_vlan_filter *vlf, *vlftmp;
4778         struct iavf_adv_rss *rss, *rsstmp;
4779         struct iavf_mac_filter *f, *ftmp;
4780         struct iavf_cloud_filter *cf, *cftmp;
4781         struct iavf_hw *hw = &adapter->hw;
4782         int err;
4783
4784         /* When reboot/shutdown is in progress no need to do anything
4785          * as the adapter is already REMOVE state that was set during
4786          * iavf_shutdown() callback.
4787          */
4788         if (adapter->state == __IAVF_REMOVE)
4789                 return;
4790
4791         set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section);
4792         /* Wait until port initialization is complete.
4793          * There are flows where register/unregister netdev may race.
4794          */
4795         while (1) {
4796                 mutex_lock(&adapter->crit_lock);
4797                 if (adapter->state == __IAVF_RUNNING ||
4798                     adapter->state == __IAVF_DOWN ||
4799                     adapter->state == __IAVF_INIT_FAILED) {
4800                         mutex_unlock(&adapter->crit_lock);
4801                         break;
4802                 }
4803
4804                 mutex_unlock(&adapter->crit_lock);
4805                 usleep_range(500, 1000);
4806         }
4807         cancel_delayed_work_sync(&adapter->watchdog_task);
4808
4809         if (adapter->netdev_registered) {
4810                 rtnl_lock();
4811                 unregister_netdevice(netdev);
4812                 adapter->netdev_registered = false;
4813                 rtnl_unlock();
4814         }
4815         if (CLIENT_ALLOWED(adapter)) {
4816                 err = iavf_lan_del_device(adapter);
4817                 if (err)
4818                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
4819                                  err);
4820         }
4821
4822         mutex_lock(&adapter->crit_lock);
4823         dev_info(&adapter->pdev->dev, "Remove device\n");
4824         iavf_change_state(adapter, __IAVF_REMOVE);
4825
4826         iavf_request_reset(adapter);
4827         msleep(50);
4828         /* If the FW isn't responding, kick it once, but only once. */
4829         if (!iavf_asq_done(hw)) {
4830                 iavf_request_reset(adapter);
4831                 msleep(50);
4832         }
4833
4834         iavf_misc_irq_disable(adapter);
4835         /* Shut down all the garbage mashers on the detention level */
4836         cancel_work_sync(&adapter->reset_task);
4837         cancel_delayed_work_sync(&adapter->watchdog_task);
4838         cancel_work_sync(&adapter->adminq_task);
4839         cancel_delayed_work_sync(&adapter->client_task);
4840
4841         adapter->aq_required = 0;
4842         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
4843
4844         iavf_free_all_tx_resources(adapter);
4845         iavf_free_all_rx_resources(adapter);
4846         iavf_free_misc_irq(adapter);
4847
4848         iavf_reset_interrupt_capability(adapter);
4849         iavf_free_q_vectors(adapter);
4850
4851         iavf_free_rss(adapter);
4852
4853         if (hw->aq.asq.count)
4854                 iavf_shutdown_adminq(hw);
4855
4856         /* destroy the locks only once, here */
4857         mutex_destroy(&hw->aq.arq_mutex);
4858         mutex_destroy(&hw->aq.asq_mutex);
4859         mutex_destroy(&adapter->client_lock);
4860         mutex_unlock(&adapter->crit_lock);
4861         mutex_destroy(&adapter->crit_lock);
4862
4863         iounmap(hw->hw_addr);
4864         pci_release_regions(pdev);
4865         iavf_free_queues(adapter);
4866         kfree(adapter->vf_res);
4867         spin_lock_bh(&adapter->mac_vlan_list_lock);
4868         /* If we got removed before an up/down sequence, we've got a filter
4869          * hanging out there that we need to get rid of.
4870          */
4871         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
4872                 list_del(&f->list);
4873                 kfree(f);
4874         }
4875         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
4876                                  list) {
4877                 list_del(&vlf->list);
4878                 kfree(vlf);
4879         }
4880
4881         spin_unlock_bh(&adapter->mac_vlan_list_lock);
4882
4883         spin_lock_bh(&adapter->cloud_filter_list_lock);
4884         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
4885                 list_del(&cf->list);
4886                 kfree(cf);
4887         }
4888         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4889
4890         spin_lock_bh(&adapter->fdir_fltr_lock);
4891         list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
4892                 list_del(&fdir->list);
4893                 kfree(fdir);
4894         }
4895         spin_unlock_bh(&adapter->fdir_fltr_lock);
4896
4897         spin_lock_bh(&adapter->adv_rss_lock);
4898         list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
4899                                  list) {
4900                 list_del(&rss->list);
4901                 kfree(rss);
4902         }
4903         spin_unlock_bh(&adapter->adv_rss_lock);
4904
4905         free_netdev(netdev);
4906
4907         pci_disable_pcie_error_reporting(pdev);
4908
4909         pci_disable_device(pdev);
4910 }
4911
4912 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
4913
4914 static struct pci_driver iavf_driver = {
4915         .name      = iavf_driver_name,
4916         .id_table  = iavf_pci_tbl,
4917         .probe     = iavf_probe,
4918         .remove    = iavf_remove,
4919         .driver.pm = &iavf_pm_ops,
4920         .shutdown  = iavf_shutdown,
4921 };
4922
4923 /**
4924  * iavf_init_module - Driver Registration Routine
4925  *
4926  * iavf_init_module is the first routine called when the driver is
4927  * loaded. All it does is register with the PCI subsystem.
4928  **/
4929 static int __init iavf_init_module(void)
4930 {
4931         pr_info("iavf: %s\n", iavf_driver_string);
4932
4933         pr_info("%s\n", iavf_copyright);
4934
4935         iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
4936                                   iavf_driver_name);
4937         if (!iavf_wq) {
4938                 pr_err("%s: Failed to create workqueue\n", iavf_driver_name);
4939                 return -ENOMEM;
4940         }
4941         return pci_register_driver(&iavf_driver);
4942 }
4943
4944 module_init(iavf_init_module);
4945
4946 /**
4947  * iavf_exit_module - Driver Exit Cleanup Routine
4948  *
4949  * iavf_exit_module is called just before the driver is removed
4950  * from memory.
4951  **/
4952 static void __exit iavf_exit_module(void)
4953 {
4954         pci_unregister_driver(&iavf_driver);
4955         destroy_workqueue(iavf_wq);
4956 }
4957
4958 module_exit(iavf_exit_module);
4959
4960 /* iavf_main.c */