drivers/net/ethernet/intel/iavf/iavf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include "iavf.h"
   5 #include "iavf_prototype.h"
   6 #include "iavf_client.h"
   7 /* All iavf tracepoints are defined by the include below, which must
   8  * be included exactly once across the whole kernel with
   9  * CREATE_TRACE_POINTS defined
  10  */
  11 #define CREATE_TRACE_POINTS
  12 #include "iavf_trace.h"
  13
  14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
  15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
  16 static int iavf_close(struct net_device *netdev);
  17 static int iavf_init_get_resources(struct iavf_adapter *adapter);
  18 static int iavf_check_reset_complete(struct iavf_hw *hw);
  19
  20 char iavf_driver_name[] = "iavf";
  21 static const char iavf_driver_string[] =
  22         "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
  23
  24 static const char iavf_copyright[] =
  25         "Copyright (c) 2013 - 2018 Intel Corporation.";
  26
  27 /* iavf_pci_tbl - PCI Device ID Table
  28  *
  29  * Wildcard entries (PCI_ANY_ID) should come last
  30  * Last entry must be all 0s
  31  *
  32  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  33  *   Class, Class Mask, private data (not used) }
  34  */
  35 static const struct pci_device_id iavf_pci_tbl[] = {
  36         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
  37         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
  38         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
  39         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
  40         /* required last entry */
  41         {0, }
  42 };
  43
  44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
  45
  46 MODULE_ALIAS("i40evf");
  47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
  49 MODULE_LICENSE("GPL v2");
  50
  51 static const struct net_device_ops iavf_netdev_ops;
  52 struct workqueue_struct *iavf_wq;
  53
  54 /**
  55  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
  56  * @hw:   pointer to the HW structure
  57  * @mem:  ptr to mem struct to fill out
  58  * @size: size of memory requested
  59  * @alignment: what to align the allocation to
  60  **/
  61 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
  62                                          struct iavf_dma_mem *mem,
  63                                          u64 size, u32 alignment)
  64 {
  65         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  66
  67         if (!mem)
  68                 return IAVF_ERR_PARAM;
  69
  70         mem->size = ALIGN(size, alignment);
  71         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
  72                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
  73         if (mem->va)
  74                 return 0;
  75         else
  76                 return IAVF_ERR_NO_MEMORY;
  77 }
  78
  79 /**
  80  * iavf_free_dma_mem_d - OS specific memory free for shared code
  81  * @hw:   pointer to the HW structure
  82  * @mem:  ptr to mem struct to free
  83  **/
  84 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw,
  85                                      struct iavf_dma_mem *mem)
  86 {
  87         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  88
  89         if (!mem || !mem->va)
  90                 return IAVF_ERR_PARAM;
  91         dma_free_coherent(&adapter->pdev->dev, mem->size,
  92                           mem->va, (dma_addr_t)mem->pa);
  93         return 0;
  94 }
  95
  96 /**
  97  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
  98  * @hw:   pointer to the HW structure
  99  * @mem:  ptr to mem struct to fill out
 100  * @size: size of memory requested
 101  **/
 102 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw,
 103                                           struct iavf_virt_mem *mem, u32 size)
 104 {
 105         if (!mem)
 106                 return IAVF_ERR_PARAM;
 107
 108         mem->size = size;
 109         mem->va = kzalloc(size, GFP_KERNEL);
 110
 111         if (mem->va)
 112                 return 0;
 113         else
 114                 return IAVF_ERR_NO_MEMORY;
 115 }
 116
 117 /**
 118  * iavf_free_virt_mem_d - OS specific memory free for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to free
 121  **/
 122 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
 123                                       struct iavf_virt_mem *mem)
 124 {
 125         if (!mem)
 126                 return IAVF_ERR_PARAM;
 127
 128         /* it's ok to kfree a NULL pointer */
 129         kfree(mem->va);
 130
 131         return 0;
 132 }
 133
 134 /**
 135  * iavf_lock_timeout - try to lock mutex but give up after timeout
 136  * @lock: mutex that should be locked
 137  * @msecs: timeout in msecs
 138  *
 139  * Returns 0 on success, negative on failure
 140  **/
 141 static int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
 142 {
 143         unsigned int wait, delay = 10;
 144
 145         for (wait = 0; wait < msecs; wait += delay) {
 146                 if (mutex_trylock(lock))
 147                         return 0;
 148
 149                 msleep(delay);
 150         }
 151
 152         return -1;
 153 }
 154
 155 /**
 156  * iavf_schedule_reset - Set the flags and schedule a reset event
 157  * @adapter: board private structure
 158  **/
 159 void iavf_schedule_reset(struct iavf_adapter *adapter)
 160 {
 161         if (!(adapter->flags &
 162               (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
 163                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
 164                 queue_work(iavf_wq, &adapter->reset_task);
 165         }
 166 }
 167
 168 /**
 169  * iavf_tx_timeout - Respond to a Tx Hang
 170  * @netdev: network interface device structure
 171  * @txqueue: queue number that is timing out
 172  **/
 173 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 174 {
 175         struct iavf_adapter *adapter = netdev_priv(netdev);
 176
 177         adapter->tx_timeout_count++;
 178         iavf_schedule_reset(adapter);
 179 }
 180
 181 /**
 182  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
 183  * @adapter: board private structure
 184  **/
 185 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
 186 {
 187         struct iavf_hw *hw = &adapter->hw;
 188
 189         if (!adapter->msix_entries)
 190                 return;
 191
 192         wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
 193
 194         iavf_flush(hw);
 195
 196         synchronize_irq(adapter->msix_entries[0].vector);
 197 }
 198
 199 /**
 200  * iavf_misc_irq_enable - Enable default interrupt generation settings
 201  * @adapter: board private structure
 202  **/
 203 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
 204 {
 205         struct iavf_hw *hw = &adapter->hw;
 206
 207         wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
 208                                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
 209         wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
 210
 211         iavf_flush(hw);
 212 }
 213
 214 /**
 215  * iavf_irq_disable - Mask off interrupt generation on the NIC
 216  * @adapter: board private structure
 217  **/
 218 static void iavf_irq_disable(struct iavf_adapter *adapter)
 219 {
 220         int i;
 221         struct iavf_hw *hw = &adapter->hw;
 222
 223         if (!adapter->msix_entries)
 224                 return;
 225
 226         for (i = 1; i < adapter->num_msix_vectors; i++) {
 227                 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
 228                 synchronize_irq(adapter->msix_entries[i].vector);
 229         }
 230         iavf_flush(hw);
 231 }
 232
 233 /**
 234  * iavf_irq_enable_queues - Enable interrupt for specified queues
 235  * @adapter: board private structure
 236  * @mask: bitmap of queues to enable
 237  **/
 238 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
 239 {
 240         struct iavf_hw *hw = &adapter->hw;
 241         int i;
 242
 243         for (i = 1; i < adapter->num_msix_vectors; i++) {
 244                 if (mask & BIT(i - 1)) {
 245                         wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
 246                              IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
 247                              IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 248                 }
 249         }
 250 }
 251
 252 /**
 253  * iavf_irq_enable - Enable default interrupt generation settings
 254  * @adapter: board private structure
 255  * @flush: boolean value whether to run rd32()
 256  **/
 257 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
 258 {
 259         struct iavf_hw *hw = &adapter->hw;
 260
 261         iavf_misc_irq_enable(adapter);
 262         iavf_irq_enable_queues(adapter, ~0);
 263
 264         if (flush)
 265                 iavf_flush(hw);
 266 }
 267
 268 /**
 269  * iavf_msix_aq - Interrupt handler for vector 0
 270  * @irq: interrupt number
 271  * @data: pointer to netdev
 272  **/
 273 static irqreturn_t iavf_msix_aq(int irq, void *data)
 274 {
 275         struct net_device *netdev = data;
 276         struct iavf_adapter *adapter = netdev_priv(netdev);
 277         struct iavf_hw *hw = &adapter->hw;
 278
 279         /* handle non-queue interrupts, these reads clear the registers */
 280         rd32(hw, IAVF_VFINT_ICR01);
 281         rd32(hw, IAVF_VFINT_ICR0_ENA1);
 282
 283         /* schedule work on the private workqueue */
 284         queue_work(iavf_wq, &adapter->adminq_task);
 285
 286         return IRQ_HANDLED;
 287 }
 288
 289 /**
 290  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
 291  * @irq: interrupt number
 292  * @data: pointer to a q_vector
 293  **/
 294 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
 295 {
 296         struct iavf_q_vector *q_vector = data;
 297
 298         if (!q_vector->tx.ring && !q_vector->rx.ring)
 299                 return IRQ_HANDLED;
 300
 301         napi_schedule_irqoff(&q_vector->napi);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 /**
 307  * iavf_map_vector_to_rxq - associate irqs with rx queues
 308  * @adapter: board private structure
 309  * @v_idx: interrupt number
 310  * @r_idx: queue number
 311  **/
 312 static void
 313 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
 314 {
 315         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 316         struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
 317         struct iavf_hw *hw = &adapter->hw;
 318
 319         rx_ring->q_vector = q_vector;
 320         rx_ring->next = q_vector->rx.ring;
 321         rx_ring->vsi = &adapter->vsi;
 322         q_vector->rx.ring = rx_ring;
 323         q_vector->rx.count++;
 324         q_vector->rx.next_update = jiffies + 1;
 325         q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
 326         q_vector->ring_mask |= BIT(r_idx);
 327         wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
 328              q_vector->rx.current_itr >> 1);
 329         q_vector->rx.current_itr = q_vector->rx.target_itr;
 330 }
 331
 332 /**
 333  * iavf_map_vector_to_txq - associate irqs with tx queues
 334  * @adapter: board private structure
 335  * @v_idx: interrupt number
 336  * @t_idx: queue number
 337  **/
 338 static void
 339 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
 340 {
 341         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 342         struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
 343         struct iavf_hw *hw = &adapter->hw;
 344
 345         tx_ring->q_vector = q_vector;
 346         tx_ring->next = q_vector->tx.ring;
 347         tx_ring->vsi = &adapter->vsi;
 348         q_vector->tx.ring = tx_ring;
 349         q_vector->tx.count++;
 350         q_vector->tx.next_update = jiffies + 1;
 351         q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
 352         q_vector->num_ringpairs++;
 353         wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
 354              q_vector->tx.target_itr >> 1);
 355         q_vector->tx.current_itr = q_vector->tx.target_itr;
 356 }
 357
 358 /**
 359  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
 360  * @adapter: board private structure to initialize
 361  *
 362  * This function maps descriptor rings to the queue-specific vectors
 363  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 364  * one vector per ring/queue, but on a constrained vector budget, we
 365  * group the rings as "efficiently" as possible.  You would add new
 366  * mapping configurations in here.
 367  **/
 368 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
 369 {
 370         int rings_remaining = adapter->num_active_queues;
 371         int ridx = 0, vidx = 0;
 372         int q_vectors;
 373
 374         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 375
 376         for (; ridx < rings_remaining; ridx++) {
 377                 iavf_map_vector_to_rxq(adapter, vidx, ridx);
 378                 iavf_map_vector_to_txq(adapter, vidx, ridx);
 379
 380                 /* In the case where we have more queues than vectors, continue
 381                  * round-robin on vectors until all queues are mapped.
 382                  */
 383                 if (++vidx >= q_vectors)
 384                         vidx = 0;
 385         }
 386
 387         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
 388 }
 389
 390 /**
 391  * iavf_irq_affinity_notify - Callback for affinity changes
 392  * @notify: context as to what irq was changed
 393  * @mask: the new affinity mask
 394  *
 395  * This is a callback function used by the irq_set_affinity_notifier function
 396  * so that we may register to receive changes to the irq affinity masks.
 397  **/
 398 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
 399                                      const cpumask_t *mask)
 400 {
 401         struct iavf_q_vector *q_vector =
 402                 container_of(notify, struct iavf_q_vector, affinity_notify);
 403
 404         cpumask_copy(&q_vector->affinity_mask, mask);
 405 }
 406
 407 /**
 408  * iavf_irq_affinity_release - Callback for affinity notifier release
 409  * @ref: internal core kernel usage
 410  *
 411  * This is a callback function used by the irq_set_affinity_notifier function
 412  * to inform the current notification subscriber that they will no longer
 413  * receive notifications.
 414  **/
 415 static void iavf_irq_affinity_release(struct kref *ref) {}
 416
 417 /**
 418  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
 419  * @adapter: board private structure
 420  * @basename: device basename
 421  *
 422  * Allocates MSI-X vectors for tx and rx handling, and requests
 423  * interrupts from the kernel.
 424  **/
 425 static int
 426 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
 427 {
 428         unsigned int vector, q_vectors;
 429         unsigned int rx_int_idx = 0, tx_int_idx = 0;
 430         int irq_num, err;
 431         int cpu;
 432
 433         iavf_irq_disable(adapter);
 434         /* Decrement for Other and TCP Timer vectors */
 435         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 436
 437         for (vector = 0; vector < q_vectors; vector++) {
 438                 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
 439
 440                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 441
 442                 if (q_vector->tx.ring && q_vector->rx.ring) {
 443                         snprintf(q_vector->name, sizeof(q_vector->name),
 444                                  "iavf-%s-TxRx-%d", basename, rx_int_idx++);
 445                         tx_int_idx++;
 446                 } else if (q_vector->rx.ring) {
 447                         snprintf(q_vector->name, sizeof(q_vector->name),
 448                                  "iavf-%s-rx-%d", basename, rx_int_idx++);
 449                 } else if (q_vector->tx.ring) {
 450                         snprintf(q_vector->name, sizeof(q_vector->name),
 451                                  "iavf-%s-tx-%d", basename, tx_int_idx++);
 452                 } else {
 453                         /* skip this unused q_vector */
 454                         continue;
 455                 }
 456                 err = request_irq(irq_num,
 457                                   iavf_msix_clean_rings,
 458                                   0,
 459                                   q_vector->name,
 460                                   q_vector);
 461                 if (err) {
 462                         dev_info(&adapter->pdev->dev,
 463                                  "Request_irq failed, error: %d\n", err);
 464                         goto free_queue_irqs;
 465                 }
 466                 /* register for affinity change notifications */
 467                 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
 468                 q_vector->affinity_notify.release =
 469                                                    iavf_irq_affinity_release;
 470                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
 471                 /* Spread the IRQ affinity hints across online CPUs. Note that
 472                  * get_cpu_mask returns a mask with a permanent lifetime so
 473                  * it's safe to use as a hint for irq_set_affinity_hint.
 474                  */
 475                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
 476                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
 477         }
 478
 479         return 0;
 480
 481 free_queue_irqs:
 482         while (vector) {
 483                 vector--;
 484                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 485                 irq_set_affinity_notifier(irq_num, NULL);
 486                 irq_set_affinity_hint(irq_num, NULL);
 487                 free_irq(irq_num, &adapter->q_vectors[vector]);
 488         }
 489         return err;
 490 }
 491
 492 /**
 493  * iavf_request_misc_irq - Initialize MSI-X interrupts
 494  * @adapter: board private structure
 495  *
 496  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 497  * vector is only for the admin queue, and stays active even when the netdev
 498  * is closed.
 499  **/
 500 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
 501 {
 502         struct net_device *netdev = adapter->netdev;
 503         int err;
 504
 505         snprintf(adapter->misc_vector_name,
 506                  sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
 507                  dev_name(&adapter->pdev->dev));
 508         err = request_irq(adapter->msix_entries[0].vector,
 509                           &iavf_msix_aq, 0,
 510                           adapter->misc_vector_name, netdev);
 511         if (err) {
 512                 dev_err(&adapter->pdev->dev,
 513                         "request_irq for %s failed: %d\n",
 514                         adapter->misc_vector_name, err);
 515                 free_irq(adapter->msix_entries[0].vector, netdev);
 516         }
 517         return err;
 518 }
 519
 520 /**
 521  * iavf_free_traffic_irqs - Free MSI-X interrupts
 522  * @adapter: board private structure
 523  *
 524  * Frees all MSI-X vectors other than 0.
 525  **/
 526 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
 527 {
 528         int vector, irq_num, q_vectors;
 529
 530         if (!adapter->msix_entries)
 531                 return;
 532
 533         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 534
 535         for (vector = 0; vector < q_vectors; vector++) {
 536                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 537                 irq_set_affinity_notifier(irq_num, NULL);
 538                 irq_set_affinity_hint(irq_num, NULL);
 539                 free_irq(irq_num, &adapter->q_vectors[vector]);
 540         }
 541 }
 542
 543 /**
 544  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
 545  * @adapter: board private structure
 546  *
 547  * Frees MSI-X vector 0.
 548  **/
 549 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
 550 {
 551         struct net_device *netdev = adapter->netdev;
 552
 553         if (!adapter->msix_entries)
 554                 return;
 555
 556         free_irq(adapter->msix_entries[0].vector, netdev);
 557 }
 558
 559 /**
 560  * iavf_configure_tx - Configure Transmit Unit after Reset
 561  * @adapter: board private structure
 562  *
 563  * Configure the Tx unit of the MAC after a reset.
 564  **/
 565 static void iavf_configure_tx(struct iavf_adapter *adapter)
 566 {
 567         struct iavf_hw *hw = &adapter->hw;
 568         int i;
 569
 570         for (i = 0; i < adapter->num_active_queues; i++)
 571                 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
 572 }
 573
 574 /**
 575  * iavf_configure_rx - Configure Receive Unit after Reset
 576  * @adapter: board private structure
 577  *
 578  * Configure the Rx unit of the MAC after a reset.
 579  **/
 580 static void iavf_configure_rx(struct iavf_adapter *adapter)
 581 {
 582         unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
 583         struct iavf_hw *hw = &adapter->hw;
 584         int i;
 585
 586         /* Legacy Rx will always default to a 2048 buffer size. */
 587 #if (PAGE_SIZE < 8192)
 588         if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
 589                 struct net_device *netdev = adapter->netdev;
 590
 591                 /* For jumbo frames on systems with 4K pages we have to use
 592                  * an order 1 page, so we might as well increase the size
 593                  * of our Rx buffer to make better use of the available space
 594                  */
 595                 rx_buf_len = IAVF_RXBUFFER_3072;
 596
 597                 /* We use a 1536 buffer size for configurations with
 598                  * standard Ethernet mtu.  On x86 this gives us enough room
 599                  * for shared info and 192 bytes of padding.
 600                  */
 601                 if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
 602                     (netdev->mtu <= ETH_DATA_LEN))
 603                         rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
 604         }
 605 #endif
 606
 607         for (i = 0; i < adapter->num_active_queues; i++) {
 608                 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
 609                 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
 610
 611                 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
 612                         clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
 613                 else
 614                         set_ring_build_skb_enabled(&adapter->rx_rings[i]);
 615         }
 616 }
 617
 618 /**
 619  * iavf_find_vlan - Search filter list for specific vlan filter
 620  * @adapter: board private structure
 621  * @vlan: vlan tag
 622  *
 623  * Returns ptr to the filter object or NULL. Must be called while holding the
 624  * mac_vlan_list_lock.
 625  **/
 626 static struct
 627 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan)
 628 {
 629         struct iavf_vlan_filter *f;
 630
 631         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 632                 if (vlan == f->vlan)
 633                         return f;
 634         }
 635         return NULL;
 636 }
 637
 638 /**
 639  * iavf_add_vlan - Add a vlan filter to the list
 640  * @adapter: board private structure
 641  * @vlan: VLAN tag
 642  *
 643  * Returns ptr to the filter object or NULL when no memory available.
 644  **/
 645 static struct
 646 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan)
 647 {
 648         struct iavf_vlan_filter *f = NULL;
 649
 650         spin_lock_bh(&adapter->mac_vlan_list_lock);
 651
 652         f = iavf_find_vlan(adapter, vlan);
 653         if (!f) {
 654                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 655                 if (!f)
 656                         goto clearout;
 657
 658                 f->vlan = vlan;
 659
 660                 list_add_tail(&f->list, &adapter->vlan_filter_list);
 661                 f->add = true;
 662                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
 663         }
 664
 665 clearout:
 666         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 667         return f;
 668 }
 669
 670 /**
 671  * iavf_del_vlan - Remove a vlan filter from the list
 672  * @adapter: board private structure
 673  * @vlan: VLAN tag
 674  **/
 675 static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan)
 676 {
 677         struct iavf_vlan_filter *f;
 678
 679         spin_lock_bh(&adapter->mac_vlan_list_lock);
 680
 681         f = iavf_find_vlan(adapter, vlan);
 682         if (f) {
 683                 f->remove = true;
 684                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
 685         }
 686
 687         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 688 }
 689
 690 /**
 691  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
 692  * @netdev: network device struct
 693  * @proto: unused protocol data
 694  * @vid: VLAN tag
 695  **/
 696 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
 697                                 __always_unused __be16 proto, u16 vid)
 698 {
 699         struct iavf_adapter *adapter = netdev_priv(netdev);
 700
 701         if (!VLAN_ALLOWED(adapter))
 702                 return -EIO;
 703         if (iavf_add_vlan(adapter, vid) == NULL)
 704                 return -ENOMEM;
 705         return 0;
 706 }
 707
 708 /**
 709  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 710  * @netdev: network device struct
 711  * @proto: unused protocol data
 712  * @vid: VLAN tag
 713  **/
 714 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
 715                                  __always_unused __be16 proto, u16 vid)
 716 {
 717         struct iavf_adapter *adapter = netdev_priv(netdev);
 718
 719         if (VLAN_ALLOWED(adapter)) {
 720                 iavf_del_vlan(adapter, vid);
 721                 return 0;
 722         }
 723         return -EIO;
 724 }
 725
 726 /**
 727  * iavf_find_filter - Search filter list for specific mac filter
 728  * @adapter: board private structure
 729  * @macaddr: the MAC address
 730  *
 731  * Returns ptr to the filter object or NULL. Must be called while holding the
 732  * mac_vlan_list_lock.
 733  **/
 734 static struct
 735 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
 736                                   const u8 *macaddr)
 737 {
 738         struct iavf_mac_filter *f;
 739
 740         if (!macaddr)
 741                 return NULL;
 742
 743         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 744                 if (ether_addr_equal(macaddr, f->macaddr))
 745                         return f;
 746         }
 747         return NULL;
 748 }
 749
 750 /**
 751  * iavf_add_filter - Add a mac filter to the filter list
 752  * @adapter: board private structure
 753  * @macaddr: the MAC address
 754  *
 755  * Returns ptr to the filter object or NULL when no memory available.
 756  **/
 757 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
 758                                         const u8 *macaddr)
 759 {
 760         struct iavf_mac_filter *f;
 761
 762         if (!macaddr)
 763                 return NULL;
 764
 765         f = iavf_find_filter(adapter, macaddr);
 766         if (!f) {
 767                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 768                 if (!f)
 769                         return f;
 770
 771                 ether_addr_copy(f->macaddr, macaddr);
 772
 773                 list_add_tail(&f->list, &adapter->mac_filter_list);
 774                 f->add = true;
 775                 f->is_new_mac = true;
 776                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
 777         } else {
 778                 f->remove = false;
 779         }
 780
 781         return f;
 782 }
 783
 784 /**
 785  * iavf_set_mac - NDO callback to set port mac address
 786  * @netdev: network interface device structure
 787  * @p: pointer to an address structure
 788  *
 789  * Returns 0 on success, negative on failure
 790  **/
 791 static int iavf_set_mac(struct net_device *netdev, void *p)
 792 {
 793         struct iavf_adapter *adapter = netdev_priv(netdev);
 794         struct iavf_hw *hw = &adapter->hw;
 795         struct iavf_mac_filter *f;
 796         struct sockaddr *addr = p;
 797
 798         if (!is_valid_ether_addr(addr->sa_data))
 799                 return -EADDRNOTAVAIL;
 800
 801         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
 802                 return 0;
 803
 804         spin_lock_bh(&adapter->mac_vlan_list_lock);
 805
 806         f = iavf_find_filter(adapter, hw->mac.addr);
 807         if (f) {
 808                 f->remove = true;
 809                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 810         }
 811
 812         f = iavf_add_filter(adapter, addr->sa_data);
 813
 814         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 815
 816         if (f) {
 817                 ether_addr_copy(hw->mac.addr, addr->sa_data);
 818         }
 819
 820         return (f == NULL) ? -ENOMEM : 0;
 821 }
 822
 823 /**
 824  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
 825  * @netdev: the netdevice
 826  * @addr: address to add
 827  *
 828  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
 829  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 830  */
 831 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
 832 {
 833         struct iavf_adapter *adapter = netdev_priv(netdev);
 834
 835         if (iavf_add_filter(adapter, addr))
 836                 return 0;
 837         else
 838                 return -ENOMEM;
 839 }
 840
 841 /**
 842  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
 843  * @netdev: the netdevice
 844  * @addr: address to add
 845  *
 846  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
 847  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 848  */
 849 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
 850 {
 851         struct iavf_adapter *adapter = netdev_priv(netdev);
 852         struct iavf_mac_filter *f;
 853
 854         /* Under some circumstances, we might receive a request to delete
 855          * our own device address from our uc list. Because we store the
 856          * device address in the VSI's MAC/VLAN filter list, we need to ignore
 857          * such requests and not delete our device address from this list.
 858          */
 859         if (ether_addr_equal(addr, netdev->dev_addr))
 860                 return 0;
 861
 862         f = iavf_find_filter(adapter, addr);
 863         if (f) {
 864                 f->remove = true;
 865                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 866         }
 867         return 0;
 868 }
 869
 870 /**
 871  * iavf_set_rx_mode - NDO callback to set the netdev filters
 872  * @netdev: network interface device structure
 873  **/
 874 static void iavf_set_rx_mode(struct net_device *netdev)
 875 {
 876         struct iavf_adapter *adapter = netdev_priv(netdev);
 877
 878         spin_lock_bh(&adapter->mac_vlan_list_lock);
 879         __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 880         __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 881         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 882
 883         if (netdev->flags & IFF_PROMISC &&
 884             !(adapter->flags & IAVF_FLAG_PROMISC_ON))
 885                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
 886         else if (!(netdev->flags & IFF_PROMISC) &&
 887                  adapter->flags & IAVF_FLAG_PROMISC_ON)
 888                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
 889
 890         if (netdev->flags & IFF_ALLMULTI &&
 891             !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
 892                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
 893         else if (!(netdev->flags & IFF_ALLMULTI) &&
 894                  adapter->flags & IAVF_FLAG_ALLMULTI_ON)
 895                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
 896 }
 897
 898 /**
 899  * iavf_napi_enable_all - enable NAPI on all queue vectors
 900  * @adapter: board private structure
 901  **/
 902 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
 903 {
 904         int q_idx;
 905         struct iavf_q_vector *q_vector;
 906         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 907
 908         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 909                 struct napi_struct *napi;
 910
 911                 q_vector = &adapter->q_vectors[q_idx];
 912                 napi = &q_vector->napi;
 913                 napi_enable(napi);
 914         }
 915 }
 916
 917 /**
 918  * iavf_napi_disable_all - disable NAPI on all queue vectors
 919  * @adapter: board private structure
 920  **/
 921 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
 922 {
 923         int q_idx;
 924         struct iavf_q_vector *q_vector;
 925         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 926
 927         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 928                 q_vector = &adapter->q_vectors[q_idx];
 929                 napi_disable(&q_vector->napi);
 930         }
 931 }
 932
 933 /**
 934  * iavf_configure - set up transmit and receive data structures
 935  * @adapter: board private structure
 936  **/
 937 static void iavf_configure(struct iavf_adapter *adapter)
 938 {
 939         struct net_device *netdev = adapter->netdev;
 940         int i;
 941
 942         iavf_set_rx_mode(netdev);
 943
 944         iavf_configure_tx(adapter);
 945         iavf_configure_rx(adapter);
 946         adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
 947
 948         for (i = 0; i < adapter->num_active_queues; i++) {
 949                 struct iavf_ring *ring = &adapter->rx_rings[i];
 950
 951                 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
 952         }
 953 }
 954
 955 /**
 956  * iavf_up_complete - Finish the last steps of bringing up a connection
 957  * @adapter: board private structure
 958  *
 959  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
 960  **/
 961 static void iavf_up_complete(struct iavf_adapter *adapter)
 962 {
 963         adapter->state = __IAVF_RUNNING;
 964         clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
 965
 966         iavf_napi_enable_all(adapter);
 967
 968         adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
 969         if (CLIENT_ENABLED(adapter))
 970                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
 971         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
 972 }
 973
 974 /**
 975  * iavf_down - Shutdown the connection processing
 976  * @adapter: board private structure
 977  *
 978  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
 979  **/
 980 void iavf_down(struct iavf_adapter *adapter)
 981 {
 982         struct net_device *netdev = adapter->netdev;
 983         struct iavf_vlan_filter *vlf;
 984         struct iavf_cloud_filter *cf;
 985         struct iavf_fdir_fltr *fdir;
 986         struct iavf_mac_filter *f;
 987         struct iavf_adv_rss *rss;
 988
 989         if (adapter->state <= __IAVF_DOWN_PENDING)
 990                 return;
 991
 992         netif_carrier_off(netdev);
 993         netif_tx_disable(netdev);
 994         adapter->link_up = false;
 995         iavf_napi_disable_all(adapter);
 996         iavf_irq_disable(adapter);
 997
 998         spin_lock_bh(&adapter->mac_vlan_list_lock);
 999
1000         /* clear the sync flag on all filters */
1001         __dev_uc_unsync(adapter->netdev, NULL);
1002         __dev_mc_unsync(adapter->netdev, NULL);
1003
1004         /* remove all MAC filters */
1005         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1006                 f->remove = true;
1007         }
1008
1009         /* remove all VLAN filters */
1010         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1011                 vlf->remove = true;
1012         }
1013
1014         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1015
1016         /* remove all cloud filters */
1017         spin_lock_bh(&adapter->cloud_filter_list_lock);
1018         list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1019                 cf->del = true;
1020         }
1021         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1022
1023         /* remove all Flow Director filters */
1024         spin_lock_bh(&adapter->fdir_fltr_lock);
1025         list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
1026                 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1027         }
1028         spin_unlock_bh(&adapter->fdir_fltr_lock);
1029
1030         /* remove all advance RSS configuration */
1031         spin_lock_bh(&adapter->adv_rss_lock);
1032         list_for_each_entry(rss, &adapter->adv_rss_list_head, list)
1033                 rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1034         spin_unlock_bh(&adapter->adv_rss_lock);
1035
1036         if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
1037             adapter->state != __IAVF_RESETTING) {
1038                 /* cancel any current operation */
1039                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1040                 /* Schedule operations to close down the HW. Don't wait
1041                  * here for this to complete. The watchdog is still running
1042                  * and it will take care of this.
1043                  */
1044                 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER;
1045                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1046                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1047                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1048                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1049                 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1050         }
1051
1052         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1053 }
1054
1055 /**
1056  * iavf_acquire_msix_vectors - Setup the MSIX capability
1057  * @adapter: board private structure
1058  * @vectors: number of vectors to request
1059  *
1060  * Work with the OS to set up the MSIX vectors needed.
1061  *
1062  * Returns 0 on success, negative on failure
1063  **/
1064 static int
1065 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1066 {
1067         int err, vector_threshold;
1068
1069         /* We'll want at least 3 (vector_threshold):
1070          * 0) Other (Admin Queue and link, mostly)
1071          * 1) TxQ[0] Cleanup
1072          * 2) RxQ[0] Cleanup
1073          */
1074         vector_threshold = MIN_MSIX_COUNT;
1075
1076         /* The more we get, the more we will assign to Tx/Rx Cleanup
1077          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1078          * Right now, we simply care about how many we'll get; we'll
1079          * set them up later while requesting irq's.
1080          */
1081         err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1082                                     vector_threshold, vectors);
1083         if (err < 0) {
1084                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1085                 kfree(adapter->msix_entries);
1086                 adapter->msix_entries = NULL;
1087                 return err;
1088         }
1089
1090         /* Adjust for only the vectors we'll use, which is minimum
1091          * of max_msix_q_vectors + NONQ_VECS, or the number of
1092          * vectors we were allocated.
1093          */
1094         adapter->num_msix_vectors = err;
1095         return 0;
1096 }
1097
1098 /**
1099  * iavf_free_queues - Free memory for all rings
1100  * @adapter: board private structure to initialize
1101  *
1102  * Free all of the memory associated with queue pairs.
1103  **/
1104 static void iavf_free_queues(struct iavf_adapter *adapter)
1105 {
1106         if (!adapter->vsi_res)
1107                 return;
1108         adapter->num_active_queues = 0;
1109         kfree(adapter->tx_rings);
1110         adapter->tx_rings = NULL;
1111         kfree(adapter->rx_rings);
1112         adapter->rx_rings = NULL;
1113 }
1114
1115 /**
1116  * iavf_alloc_queues - Allocate memory for all rings
1117  * @adapter: board private structure to initialize
1118  *
1119  * We allocate one ring per queue at run-time since we don't know the
1120  * number of queues at compile-time.  The polling_netdev array is
1121  * intended for Multiqueue, but should work fine with a single queue.
1122  **/
1123 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1124 {
1125         int i, num_active_queues;
1126
1127         /* If we're in reset reallocating queues we don't actually know yet for
1128          * certain the PF gave us the number of queues we asked for but we'll
1129          * assume it did.  Once basic reset is finished we'll confirm once we
1130          * start negotiating config with PF.
1131          */
1132         if (adapter->num_req_queues)
1133                 num_active_queues = adapter->num_req_queues;
1134         else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1135                  adapter->num_tc)
1136                 num_active_queues = adapter->ch_config.total_qps;
1137         else
1138                 num_active_queues = min_t(int,
1139                                           adapter->vsi_res->num_queue_pairs,
1140                                           (int)(num_online_cpus()));
1141
1142
1143         adapter->tx_rings = kcalloc(num_active_queues,
1144                                     sizeof(struct iavf_ring), GFP_KERNEL);
1145         if (!adapter->tx_rings)
1146                 goto err_out;
1147         adapter->rx_rings = kcalloc(num_active_queues,
1148                                     sizeof(struct iavf_ring), GFP_KERNEL);
1149         if (!adapter->rx_rings)
1150                 goto err_out;
1151
1152         for (i = 0; i < num_active_queues; i++) {
1153                 struct iavf_ring *tx_ring;
1154                 struct iavf_ring *rx_ring;
1155
1156                 tx_ring = &adapter->tx_rings[i];
1157
1158                 tx_ring->queue_index = i;
1159                 tx_ring->netdev = adapter->netdev;
1160                 tx_ring->dev = &adapter->pdev->dev;
1161                 tx_ring->count = adapter->tx_desc_count;
1162                 tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1163                 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1164                         tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1165
1166                 rx_ring = &adapter->rx_rings[i];
1167                 rx_ring->queue_index = i;
1168                 rx_ring->netdev = adapter->netdev;
1169                 rx_ring->dev = &adapter->pdev->dev;
1170                 rx_ring->count = adapter->rx_desc_count;
1171                 rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1172         }
1173
1174         adapter->num_active_queues = num_active_queues;
1175
1176         return 0;
1177
1178 err_out:
1179         iavf_free_queues(adapter);
1180         return -ENOMEM;
1181 }
1182
1183 /**
1184  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1185  * @adapter: board private structure to initialize
1186  *
1187  * Attempt to configure the interrupts using the best available
1188  * capabilities of the hardware and the kernel.
1189  **/
1190 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1191 {
1192         int vector, v_budget;
1193         int pairs = 0;
1194         int err = 0;
1195
1196         if (!adapter->vsi_res) {
1197                 err = -EIO;
1198                 goto out;
1199         }
1200         pairs = adapter->num_active_queues;
1201
1202         /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1203          * us much good if we have more vectors than CPUs. However, we already
1204          * limit the total number of queues by the number of CPUs so we do not
1205          * need any further limiting here.
1206          */
1207         v_budget = min_t(int, pairs + NONQ_VECS,
1208                          (int)adapter->vf_res->max_vectors);
1209
1210         adapter->msix_entries = kcalloc(v_budget,
1211                                         sizeof(struct msix_entry), GFP_KERNEL);
1212         if (!adapter->msix_entries) {
1213                 err = -ENOMEM;
1214                 goto out;
1215         }
1216
1217         for (vector = 0; vector < v_budget; vector++)
1218                 adapter->msix_entries[vector].entry = vector;
1219
1220         err = iavf_acquire_msix_vectors(adapter, v_budget);
1221
1222 out:
1223         netif_set_real_num_rx_queues(adapter->netdev, pairs);
1224         netif_set_real_num_tx_queues(adapter->netdev, pairs);
1225         return err;
1226 }
1227
1228 /**
1229  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1230  * @adapter: board private structure
1231  *
1232  * Return 0 on success, negative on failure
1233  **/
1234 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1235 {
1236         struct iavf_aqc_get_set_rss_key_data *rss_key =
1237                 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1238         struct iavf_hw *hw = &adapter->hw;
1239         int ret = 0;
1240
1241         if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1242                 /* bail because we already have a command pending */
1243                 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1244                         adapter->current_op);
1245                 return -EBUSY;
1246         }
1247
1248         ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1249         if (ret) {
1250                 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1251                         iavf_stat_str(hw, ret),
1252                         iavf_aq_str(hw, hw->aq.asq_last_status));
1253                 return ret;
1254
1255         }
1256
1257         ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1258                                   adapter->rss_lut, adapter->rss_lut_size);
1259         if (ret) {
1260                 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1261                         iavf_stat_str(hw, ret),
1262                         iavf_aq_str(hw, hw->aq.asq_last_status));
1263         }
1264
1265         return ret;
1266
1267 }
1268
1269 /**
1270  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1271  * @adapter: board private structure
1272  *
1273  * Returns 0 on success, negative on failure
1274  **/
1275 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1276 {
1277         struct iavf_hw *hw = &adapter->hw;
1278         u32 *dw;
1279         u16 i;
1280
1281         dw = (u32 *)adapter->rss_key;
1282         for (i = 0; i <= adapter->rss_key_size / 4; i++)
1283                 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1284
1285         dw = (u32 *)adapter->rss_lut;
1286         for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1287                 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1288
1289         iavf_flush(hw);
1290
1291         return 0;
1292 }
1293
1294 /**
1295  * iavf_config_rss - Configure RSS keys and lut
1296  * @adapter: board private structure
1297  *
1298  * Returns 0 on success, negative on failure
1299  **/
1300 int iavf_config_rss(struct iavf_adapter *adapter)
1301 {
1302
1303         if (RSS_PF(adapter)) {
1304                 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1305                                         IAVF_FLAG_AQ_SET_RSS_KEY;
1306                 return 0;
1307         } else if (RSS_AQ(adapter)) {
1308                 return iavf_config_rss_aq(adapter);
1309         } else {
1310                 return iavf_config_rss_reg(adapter);
1311         }
1312 }
1313
1314 /**
1315  * iavf_fill_rss_lut - Fill the lut with default values
1316  * @adapter: board private structure
1317  **/
1318 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1319 {
1320         u16 i;
1321
1322         for (i = 0; i < adapter->rss_lut_size; i++)
1323                 adapter->rss_lut[i] = i % adapter->num_active_queues;
1324 }
1325
1326 /**
1327  * iavf_init_rss - Prepare for RSS
1328  * @adapter: board private structure
1329  *
1330  * Return 0 on success, negative on failure
1331  **/
1332 static int iavf_init_rss(struct iavf_adapter *adapter)
1333 {
1334         struct iavf_hw *hw = &adapter->hw;
1335         int ret;
1336
1337         if (!RSS_PF(adapter)) {
1338                 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1339                 if (adapter->vf_res->vf_cap_flags &
1340                     VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1341                         adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1342                 else
1343                         adapter->hena = IAVF_DEFAULT_RSS_HENA;
1344
1345                 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1346                 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1347         }
1348
1349         iavf_fill_rss_lut(adapter);
1350         netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1351         ret = iavf_config_rss(adapter);
1352
1353         return ret;
1354 }
1355
1356 /**
1357  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1358  * @adapter: board private structure to initialize
1359  *
1360  * We allocate one q_vector per queue interrupt.  If allocation fails we
1361  * return -ENOMEM.
1362  **/
1363 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1364 {
1365         int q_idx = 0, num_q_vectors;
1366         struct iavf_q_vector *q_vector;
1367
1368         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1369         adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1370                                      GFP_KERNEL);
1371         if (!adapter->q_vectors)
1372                 return -ENOMEM;
1373
1374         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1375                 q_vector = &adapter->q_vectors[q_idx];
1376                 q_vector->adapter = adapter;
1377                 q_vector->vsi = &adapter->vsi;
1378                 q_vector->v_idx = q_idx;
1379                 q_vector->reg_idx = q_idx;
1380                 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1381                 netif_napi_add(adapter->netdev, &q_vector->napi,
1382                                iavf_napi_poll, NAPI_POLL_WEIGHT);
1383         }
1384
1385         return 0;
1386 }
1387
1388 /**
1389  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1390  * @adapter: board private structure to initialize
1391  *
1392  * This function frees the memory allocated to the q_vectors.  In addition if
1393  * NAPI is enabled it will delete any references to the NAPI struct prior
1394  * to freeing the q_vector.
1395  **/
1396 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1397 {
1398         int q_idx, num_q_vectors;
1399         int napi_vectors;
1400
1401         if (!adapter->q_vectors)
1402                 return;
1403
1404         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1405         napi_vectors = adapter->num_active_queues;
1406
1407         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1408                 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1409
1410                 if (q_idx < napi_vectors)
1411                         netif_napi_del(&q_vector->napi);
1412         }
1413         kfree(adapter->q_vectors);
1414         adapter->q_vectors = NULL;
1415 }
1416
1417 /**
1418  * iavf_reset_interrupt_capability - Reset MSIX setup
1419  * @adapter: board private structure
1420  *
1421  **/
1422 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1423 {
1424         if (!adapter->msix_entries)
1425                 return;
1426
1427         pci_disable_msix(adapter->pdev);
1428         kfree(adapter->msix_entries);
1429         adapter->msix_entries = NULL;
1430 }
1431
1432 /**
1433  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1434  * @adapter: board private structure to initialize
1435  *
1436  **/
1437 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1438 {
1439         int err;
1440
1441         err = iavf_alloc_queues(adapter);
1442         if (err) {
1443                 dev_err(&adapter->pdev->dev,
1444                         "Unable to allocate memory for queues\n");
1445                 goto err_alloc_queues;
1446         }
1447
1448         rtnl_lock();
1449         err = iavf_set_interrupt_capability(adapter);
1450         rtnl_unlock();
1451         if (err) {
1452                 dev_err(&adapter->pdev->dev,
1453                         "Unable to setup interrupt capabilities\n");
1454                 goto err_set_interrupt;
1455         }
1456
1457         err = iavf_alloc_q_vectors(adapter);
1458         if (err) {
1459                 dev_err(&adapter->pdev->dev,
1460                         "Unable to allocate memory for queue vectors\n");
1461                 goto err_alloc_q_vectors;
1462         }
1463
1464         /* If we've made it so far while ADq flag being ON, then we haven't
1465          * bailed out anywhere in middle. And ADq isn't just enabled but actual
1466          * resources have been allocated in the reset path.
1467          * Now we can truly claim that ADq is enabled.
1468          */
1469         if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1470             adapter->num_tc)
1471                 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1472                          adapter->num_tc);
1473
1474         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1475                  (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1476                  adapter->num_active_queues);
1477
1478         return 0;
1479 err_alloc_q_vectors:
1480         iavf_reset_interrupt_capability(adapter);
1481 err_set_interrupt:
1482         iavf_free_queues(adapter);
1483 err_alloc_queues:
1484         return err;
1485 }
1486
1487 /**
1488  * iavf_free_rss - Free memory used by RSS structs
1489  * @adapter: board private structure
1490  **/
1491 static void iavf_free_rss(struct iavf_adapter *adapter)
1492 {
1493         kfree(adapter->rss_key);
1494         adapter->rss_key = NULL;
1495
1496         kfree(adapter->rss_lut);
1497         adapter->rss_lut = NULL;
1498 }
1499
1500 /**
1501  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1502  * @adapter: board private structure
1503  *
1504  * Returns 0 on success, negative on failure
1505  **/
1506 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1507 {
1508         struct net_device *netdev = adapter->netdev;
1509         int err;
1510
1511         if (netif_running(netdev))
1512                 iavf_free_traffic_irqs(adapter);
1513         iavf_free_misc_irq(adapter);
1514         iavf_reset_interrupt_capability(adapter);
1515         iavf_free_q_vectors(adapter);
1516         iavf_free_queues(adapter);
1517
1518         err =  iavf_init_interrupt_scheme(adapter);
1519         if (err)
1520                 goto err;
1521
1522         netif_tx_stop_all_queues(netdev);
1523
1524         err = iavf_request_misc_irq(adapter);
1525         if (err)
1526                 goto err;
1527
1528         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1529
1530         iavf_map_rings_to_vectors(adapter);
1531 err:
1532         return err;
1533 }
1534
1535 /**
1536  * iavf_process_aq_command - process aq_required flags
1537  * and sends aq command
1538  * @adapter: pointer to iavf adapter structure
1539  *
1540  * Returns 0 on success
1541  * Returns error code if no command was sent
1542  * or error code if the command failed.
1543  **/
1544 static int iavf_process_aq_command(struct iavf_adapter *adapter)
1545 {
1546         if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
1547                 return iavf_send_vf_config_msg(adapter);
1548         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
1549                 iavf_disable_queues(adapter);
1550                 return 0;
1551         }
1552
1553         if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
1554                 iavf_map_queues(adapter);
1555                 return 0;
1556         }
1557
1558         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
1559                 iavf_add_ether_addrs(adapter);
1560                 return 0;
1561         }
1562
1563         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
1564                 iavf_add_vlans(adapter);
1565                 return 0;
1566         }
1567
1568         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
1569                 iavf_del_ether_addrs(adapter);
1570                 return 0;
1571         }
1572
1573         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
1574                 iavf_del_vlans(adapter);
1575                 return 0;
1576         }
1577
1578         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
1579                 iavf_enable_vlan_stripping(adapter);
1580                 return 0;
1581         }
1582
1583         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
1584                 iavf_disable_vlan_stripping(adapter);
1585                 return 0;
1586         }
1587
1588         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
1589                 iavf_configure_queues(adapter);
1590                 return 0;
1591         }
1592
1593         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
1594                 iavf_enable_queues(adapter);
1595                 return 0;
1596         }
1597
1598         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
1599                 /* This message goes straight to the firmware, not the
1600                  * PF, so we don't have to set current_op as we will
1601                  * not get a response through the ARQ.
1602                  */
1603                 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
1604                 return 0;
1605         }
1606         if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
1607                 iavf_get_hena(adapter);
1608                 return 0;
1609         }
1610         if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
1611                 iavf_set_hena(adapter);
1612                 return 0;
1613         }
1614         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
1615                 iavf_set_rss_key(adapter);
1616                 return 0;
1617         }
1618         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
1619                 iavf_set_rss_lut(adapter);
1620                 return 0;
1621         }
1622
1623         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
1624                 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
1625                                        FLAG_VF_MULTICAST_PROMISC);
1626                 return 0;
1627         }
1628
1629         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
1630                 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
1631                 return 0;
1632         }
1633
1634         if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) &&
1635             (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
1636                 iavf_set_promiscuous(adapter, 0);
1637                 return 0;
1638         }
1639
1640         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
1641                 iavf_enable_channels(adapter);
1642                 return 0;
1643         }
1644
1645         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
1646                 iavf_disable_channels(adapter);
1647                 return 0;
1648         }
1649         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1650                 iavf_add_cloud_filter(adapter);
1651                 return 0;
1652         }
1653
1654         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1655                 iavf_del_cloud_filter(adapter);
1656                 return 0;
1657         }
1658         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1659                 iavf_del_cloud_filter(adapter);
1660                 return 0;
1661         }
1662         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1663                 iavf_add_cloud_filter(adapter);
1664                 return 0;
1665         }
1666         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
1667                 iavf_add_fdir_filter(adapter);
1668                 return IAVF_SUCCESS;
1669         }
1670         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
1671                 iavf_del_fdir_filter(adapter);
1672                 return IAVF_SUCCESS;
1673         }
1674         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
1675                 iavf_add_adv_rss_cfg(adapter);
1676                 return 0;
1677         }
1678         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
1679                 iavf_del_adv_rss_cfg(adapter);
1680                 return 0;
1681         }
1682         return -EAGAIN;
1683 }
1684
1685 /**
1686  * iavf_startup - first step of driver startup
1687  * @adapter: board private structure
1688  *
1689  * Function process __IAVF_STARTUP driver state.
1690  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
1691  * when fails it returns -EAGAIN
1692  **/
1693 static int iavf_startup(struct iavf_adapter *adapter)
1694 {
1695         struct pci_dev *pdev = adapter->pdev;
1696         struct iavf_hw *hw = &adapter->hw;
1697         int err;
1698
1699         WARN_ON(adapter->state != __IAVF_STARTUP);
1700
1701         /* driver loaded, probe complete */
1702         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
1703         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
1704         err = iavf_set_mac_type(hw);
1705         if (err) {
1706                 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", err);
1707                 goto err;
1708         }
1709
1710         err = iavf_check_reset_complete(hw);
1711         if (err) {
1712                 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
1713                          err);
1714                 goto err;
1715         }
1716         hw->aq.num_arq_entries = IAVF_AQ_LEN;
1717         hw->aq.num_asq_entries = IAVF_AQ_LEN;
1718         hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
1719         hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
1720
1721         err = iavf_init_adminq(hw);
1722         if (err) {
1723                 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", err);
1724                 goto err;
1725         }
1726         err = iavf_send_api_ver(adapter);
1727         if (err) {
1728                 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err);
1729                 iavf_shutdown_adminq(hw);
1730                 goto err;
1731         }
1732         adapter->state = __IAVF_INIT_VERSION_CHECK;
1733 err:
1734         return err;
1735 }
1736
1737 /**
1738  * iavf_init_version_check - second step of driver startup
1739  * @adapter: board private structure
1740  *
1741  * Function process __IAVF_INIT_VERSION_CHECK driver state.
1742  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
1743  * when fails it returns -EAGAIN
1744  **/
1745 static int iavf_init_version_check(struct iavf_adapter *adapter)
1746 {
1747         struct pci_dev *pdev = adapter->pdev;
1748         struct iavf_hw *hw = &adapter->hw;
1749         int err = -EAGAIN;
1750
1751         WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
1752
1753         if (!iavf_asq_done(hw)) {
1754                 dev_err(&pdev->dev, "Admin queue command never completed\n");
1755                 iavf_shutdown_adminq(hw);
1756                 adapter->state = __IAVF_STARTUP;
1757                 goto err;
1758         }
1759
1760         /* aq msg sent, awaiting reply */
1761         err = iavf_verify_api_ver(adapter);
1762         if (err) {
1763                 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK)
1764                         err = iavf_send_api_ver(adapter);
1765                 else
1766                         dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
1767                                 adapter->pf_version.major,
1768                                 adapter->pf_version.minor,
1769                                 VIRTCHNL_VERSION_MAJOR,
1770                                 VIRTCHNL_VERSION_MINOR);
1771                 goto err;
1772         }
1773         err = iavf_send_vf_config_msg(adapter);
1774         if (err) {
1775                 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
1776                         err);
1777                 goto err;
1778         }
1779         adapter->state = __IAVF_INIT_GET_RESOURCES;
1780
1781 err:
1782         return err;
1783 }
1784
1785 /**
1786  * iavf_init_get_resources - third step of driver startup
1787  * @adapter: board private structure
1788  *
1789  * Function process __IAVF_INIT_GET_RESOURCES driver state and
1790  * finishes driver initialization procedure.
1791  * When success the state is changed to __IAVF_DOWN
1792  * when fails it returns -EAGAIN
1793  **/
1794 static int iavf_init_get_resources(struct iavf_adapter *adapter)
1795 {
1796         struct net_device *netdev = adapter->netdev;
1797         struct pci_dev *pdev = adapter->pdev;
1798         struct iavf_hw *hw = &adapter->hw;
1799         int err;
1800
1801         WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
1802         /* aq msg sent, awaiting reply */
1803         if (!adapter->vf_res) {
1804                 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
1805                                           GFP_KERNEL);
1806                 if (!adapter->vf_res) {
1807                         err = -ENOMEM;
1808                         goto err;
1809                 }
1810         }
1811         err = iavf_get_vf_config(adapter);
1812         if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) {
1813                 err = iavf_send_vf_config_msg(adapter);
1814                 goto err;
1815         } else if (err == IAVF_ERR_PARAM) {
1816                 /* We only get ERR_PARAM if the device is in a very bad
1817                  * state or if we've been disabled for previous bad
1818                  * behavior. Either way, we're done now.
1819                  */
1820                 iavf_shutdown_adminq(hw);
1821                 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
1822                 return 0;
1823         }
1824         if (err) {
1825                 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
1826                 goto err_alloc;
1827         }
1828
1829         err = iavf_process_config(adapter);
1830         if (err)
1831                 goto err_alloc;
1832         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1833
1834         adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
1835
1836         netdev->netdev_ops = &iavf_netdev_ops;
1837         iavf_set_ethtool_ops(netdev);
1838         netdev->watchdog_timeo = 5 * HZ;
1839
1840         /* MTU range: 68 - 9710 */
1841         netdev->min_mtu = ETH_MIN_MTU;
1842         netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
1843
1844         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
1845                 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
1846                          adapter->hw.mac.addr);
1847                 eth_hw_addr_random(netdev);
1848                 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
1849         } else {
1850                 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
1851                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
1852         }
1853
1854         adapter->tx_desc_count = IAVF_DEFAULT_TXD;
1855         adapter->rx_desc_count = IAVF_DEFAULT_RXD;
1856         err = iavf_init_interrupt_scheme(adapter);
1857         if (err)
1858                 goto err_sw_init;
1859         iavf_map_rings_to_vectors(adapter);
1860         if (adapter->vf_res->vf_cap_flags &
1861                 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
1862                 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
1863
1864         err = iavf_request_misc_irq(adapter);
1865         if (err)
1866                 goto err_sw_init;
1867
1868         netif_carrier_off(netdev);
1869         adapter->link_up = false;
1870
1871         /* set the semaphore to prevent any callbacks after device registration
1872          * up to time when state of driver will be set to __IAVF_DOWN
1873          */
1874         rtnl_lock();
1875         if (!adapter->netdev_registered) {
1876                 err = register_netdevice(netdev);
1877                 if (err) {
1878                         rtnl_unlock();
1879                         goto err_register;
1880                 }
1881         }
1882
1883         adapter->netdev_registered = true;
1884
1885         netif_tx_stop_all_queues(netdev);
1886         if (CLIENT_ALLOWED(adapter)) {
1887                 err = iavf_lan_add_device(adapter);
1888                 if (err)
1889                         dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
1890                                  err);
1891         }
1892         dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
1893         if (netdev->features & NETIF_F_GRO)
1894                 dev_info(&pdev->dev, "GRO is enabled\n");
1895
1896         adapter->state = __IAVF_DOWN;
1897         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1898         rtnl_unlock();
1899
1900         iavf_misc_irq_enable(adapter);
1901         wake_up(&adapter->down_waitqueue);
1902
1903         adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
1904         adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
1905         if (!adapter->rss_key || !adapter->rss_lut) {
1906                 err = -ENOMEM;
1907                 goto err_mem;
1908         }
1909         if (RSS_AQ(adapter))
1910                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
1911         else
1912                 iavf_init_rss(adapter);
1913
1914         return err;
1915 err_mem:
1916         iavf_free_rss(adapter);
1917 err_register:
1918         iavf_free_misc_irq(adapter);
1919 err_sw_init:
1920         iavf_reset_interrupt_capability(adapter);
1921 err_alloc:
1922         kfree(adapter->vf_res);
1923         adapter->vf_res = NULL;
1924 err:
1925         return err;
1926 }
1927
1928 /**
1929  * iavf_watchdog_task - Periodic call-back task
1930  * @work: pointer to work_struct
1931  **/
1932 static void iavf_watchdog_task(struct work_struct *work)
1933 {
1934         struct iavf_adapter *adapter = container_of(work,
1935                                                     struct iavf_adapter,
1936                                                     watchdog_task.work);
1937         struct iavf_hw *hw = &adapter->hw;
1938         u32 reg_val;
1939
1940         if (!mutex_trylock(&adapter->crit_lock))
1941                 goto restart_watchdog;
1942
1943         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
1944                 adapter->state = __IAVF_COMM_FAILED;
1945
1946         switch (adapter->state) {
1947         case __IAVF_COMM_FAILED:
1948                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
1949                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
1950                 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
1951                     reg_val == VIRTCHNL_VFR_COMPLETED) {
1952                         /* A chance for redemption! */
1953                         dev_err(&adapter->pdev->dev,
1954                                 "Hardware came out of reset. Attempting reinit.\n");
1955                         adapter->state = __IAVF_STARTUP;
1956                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
1957                         queue_delayed_work(iavf_wq, &adapter->init_task, 10);
1958                         mutex_unlock(&adapter->crit_lock);
1959                         /* Don't reschedule the watchdog, since we've restarted
1960                          * the init task. When init_task contacts the PF and
1961                          * gets everything set up again, it'll restart the
1962                          * watchdog for us. Down, boy. Sit. Stay. Woof.
1963                          */
1964                         return;
1965                 }
1966                 adapter->aq_required = 0;
1967                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1968                 queue_delayed_work(iavf_wq,
1969                                    &adapter->watchdog_task,
1970                                    msecs_to_jiffies(10));
1971                 goto watchdog_done;
1972         case __IAVF_RESETTING:
1973                 mutex_unlock(&adapter->crit_lock);
1974                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
1975                 return;
1976         case __IAVF_DOWN:
1977         case __IAVF_DOWN_PENDING:
1978         case __IAVF_TESTING:
1979         case __IAVF_RUNNING:
1980                 if (adapter->current_op) {
1981                         if (!iavf_asq_done(hw)) {
1982                                 dev_dbg(&adapter->pdev->dev,
1983                                         "Admin queue timeout\n");
1984                                 iavf_send_api_ver(adapter);
1985                         }
1986                 } else {
1987                         /* An error will be returned if no commands were
1988                          * processed; use this opportunity to update stats
1989                          */
1990                         if (iavf_process_aq_command(adapter) &&
1991                             adapter->state == __IAVF_RUNNING)
1992                                 iavf_request_stats(adapter);
1993                 }
1994                 break;
1995         case __IAVF_REMOVE:
1996                 mutex_unlock(&adapter->crit_lock);
1997                 return;
1998         default:
1999                 goto restart_watchdog;
2000         }
2001
2002                 /* check for hw reset */
2003         reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2004         if (!reg_val) {
2005                 adapter->flags |= IAVF_FLAG_RESET_PENDING;
2006                 adapter->aq_required = 0;
2007                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2008                 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2009                 queue_work(iavf_wq, &adapter->reset_task);
2010                 goto watchdog_done;
2011         }
2012
2013         schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2014 watchdog_done:
2015         if (adapter->state == __IAVF_RUNNING ||
2016             adapter->state == __IAVF_COMM_FAILED)
2017                 iavf_detect_recover_hung(&adapter->vsi);
2018         mutex_unlock(&adapter->crit_lock);
2019 restart_watchdog:
2020         if (adapter->aq_required)
2021                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2022                                    msecs_to_jiffies(20));
2023         else
2024                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2025         queue_work(iavf_wq, &adapter->adminq_task);
2026 }
2027
2028 static void iavf_disable_vf(struct iavf_adapter *adapter)
2029 {
2030         struct iavf_mac_filter *f, *ftmp;
2031         struct iavf_vlan_filter *fv, *fvtmp;
2032         struct iavf_cloud_filter *cf, *cftmp;
2033
2034         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2035
2036         /* We don't use netif_running() because it may be true prior to
2037          * ndo_open() returning, so we can't assume it means all our open
2038          * tasks have finished, since we're not holding the rtnl_lock here.
2039          */
2040         if (adapter->state == __IAVF_RUNNING) {
2041                 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2042                 netif_carrier_off(adapter->netdev);
2043                 netif_tx_disable(adapter->netdev);
2044                 adapter->link_up = false;
2045                 iavf_napi_disable_all(adapter);
2046                 iavf_irq_disable(adapter);
2047                 iavf_free_traffic_irqs(adapter);
2048                 iavf_free_all_tx_resources(adapter);
2049                 iavf_free_all_rx_resources(adapter);
2050         }
2051
2052         spin_lock_bh(&adapter->mac_vlan_list_lock);
2053
2054         /* Delete all of the filters */
2055         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2056                 list_del(&f->list);
2057                 kfree(f);
2058         }
2059
2060         list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2061                 list_del(&fv->list);
2062                 kfree(fv);
2063         }
2064
2065         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2066
2067         spin_lock_bh(&adapter->cloud_filter_list_lock);
2068         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2069                 list_del(&cf->list);
2070                 kfree(cf);
2071                 adapter->num_cloud_filters--;
2072         }
2073         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2074
2075         iavf_free_misc_irq(adapter);
2076         iavf_reset_interrupt_capability(adapter);
2077         iavf_free_queues(adapter);
2078         iavf_free_q_vectors(adapter);
2079         memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2080         iavf_shutdown_adminq(&adapter->hw);
2081         adapter->netdev->flags &= ~IFF_UP;
2082         mutex_unlock(&adapter->crit_lock);
2083         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2084         adapter->state = __IAVF_DOWN;
2085         wake_up(&adapter->down_waitqueue);
2086         dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2087 }
2088
2089 /**
2090  * iavf_reset_task - Call-back task to handle hardware reset
2091  * @work: pointer to work_struct
2092  *
2093  * During reset we need to shut down and reinitialize the admin queue
2094  * before we can use it to communicate with the PF again. We also clear
2095  * and reinit the rings because that context is lost as well.
2096  **/
2097 static void iavf_reset_task(struct work_struct *work)
2098 {
2099         struct iavf_adapter *adapter = container_of(work,
2100                                                       struct iavf_adapter,
2101                                                       reset_task);
2102         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2103         struct net_device *netdev = adapter->netdev;
2104         struct iavf_hw *hw = &adapter->hw;
2105         struct iavf_mac_filter *f, *ftmp;
2106         struct iavf_vlan_filter *vlf;
2107         struct iavf_cloud_filter *cf;
2108         u32 reg_val;
2109         int i = 0, err;
2110         bool running;
2111
2112         /* When device is being removed it doesn't make sense to run the reset
2113          * task, just return in such a case.
2114          */
2115         if (mutex_is_locked(&adapter->remove_lock))
2116                 return;
2117
2118         if (iavf_lock_timeout(&adapter->crit_lock, 200)) {
2119                 schedule_work(&adapter->reset_task);
2120                 return;
2121         }
2122         while (!mutex_trylock(&adapter->client_lock))
2123                 usleep_range(500, 1000);
2124         if (CLIENT_ENABLED(adapter)) {
2125                 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
2126                                     IAVF_FLAG_CLIENT_NEEDS_CLOSE |
2127                                     IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
2128                                     IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
2129                 cancel_delayed_work_sync(&adapter->client_task);
2130                 iavf_notify_client_close(&adapter->vsi, true);
2131         }
2132         iavf_misc_irq_disable(adapter);
2133         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2134                 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
2135                 /* Restart the AQ here. If we have been reset but didn't
2136                  * detect it, or if the PF had to reinit, our AQ will be hosed.
2137                  */
2138                 iavf_shutdown_adminq(hw);
2139                 iavf_init_adminq(hw);
2140                 iavf_request_reset(adapter);
2141         }
2142         adapter->flags |= IAVF_FLAG_RESET_PENDING;
2143
2144         /* poll until we see the reset actually happen */
2145         for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
2146                 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
2147                           IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2148                 if (!reg_val)
2149                         break;
2150                 usleep_range(5000, 10000);
2151         }
2152         if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
2153                 dev_info(&adapter->pdev->dev, "Never saw reset\n");
2154                 goto continue_reset; /* act like the reset happened */
2155         }
2156
2157         /* wait until the reset is complete and the PF is responding to us */
2158         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
2159                 /* sleep first to make sure a minimum wait time is met */
2160                 msleep(IAVF_RESET_WAIT_MS);
2161
2162                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2163                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2164                 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
2165                         break;
2166         }
2167
2168         pci_set_master(adapter->pdev);
2169
2170         if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
2171                 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
2172                         reg_val);
2173                 iavf_disable_vf(adapter);
2174                 mutex_unlock(&adapter->client_lock);
2175                 return; /* Do not attempt to reinit. It's dead, Jim. */
2176         }
2177
2178 continue_reset:
2179         /* We don't use netif_running() because it may be true prior to
2180          * ndo_open() returning, so we can't assume it means all our open
2181          * tasks have finished, since we're not holding the rtnl_lock here.
2182          */
2183         running = ((adapter->state == __IAVF_RUNNING) ||
2184                    (adapter->state == __IAVF_RESETTING));
2185
2186         if (running) {
2187                 netif_carrier_off(netdev);
2188                 netif_tx_stop_all_queues(netdev);
2189                 adapter->link_up = false;
2190                 iavf_napi_disable_all(adapter);
2191         }
2192         iavf_irq_disable(adapter);
2193
2194         adapter->state = __IAVF_RESETTING;
2195         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2196
2197         /* free the Tx/Rx rings and descriptors, might be better to just
2198          * re-use them sometime in the future
2199          */
2200         iavf_free_all_rx_resources(adapter);
2201         iavf_free_all_tx_resources(adapter);
2202
2203         adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
2204         /* kill and reinit the admin queue */
2205         iavf_shutdown_adminq(hw);
2206         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2207         err = iavf_init_adminq(hw);
2208         if (err)
2209                 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
2210                          err);
2211         adapter->aq_required = 0;
2212
2213         if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
2214                 err = iavf_reinit_interrupt_scheme(adapter);
2215                 if (err)
2216                         goto reset_err;
2217         }
2218
2219         if (RSS_AQ(adapter)) {
2220                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2221         } else {
2222                 err = iavf_init_rss(adapter);
2223                 if (err)
2224                         goto reset_err;
2225         }
2226
2227         adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
2228         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
2229
2230         spin_lock_bh(&adapter->mac_vlan_list_lock);
2231
2232         /* Delete filter for the current MAC address, it could have
2233          * been changed by the PF via administratively set MAC.
2234          * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
2235          */
2236         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2237                 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
2238                         list_del(&f->list);
2239                         kfree(f);
2240                 }
2241         }
2242         /* re-add all MAC filters */
2243         list_for_each_entry(f, &adapter->mac_filter_list, list) {
2244                 f->add = true;
2245         }
2246         /* re-add all VLAN filters */
2247         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
2248                 vlf->add = true;
2249         }
2250
2251         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2252
2253         /* check if TCs are running and re-add all cloud filters */
2254         spin_lock_bh(&adapter->cloud_filter_list_lock);
2255         if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
2256             adapter->num_tc) {
2257                 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
2258                         cf->add = true;
2259                 }
2260         }
2261         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2262
2263         adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
2264         adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
2265         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
2266         iavf_misc_irq_enable(adapter);
2267
2268         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2);
2269
2270         /* We were running when the reset started, so we need to restore some
2271          * state here.
2272          */
2273         if (running) {
2274                 /* allocate transmit descriptors */
2275                 err = iavf_setup_all_tx_resources(adapter);
2276                 if (err)
2277                         goto reset_err;
2278
2279                 /* allocate receive descriptors */
2280                 err = iavf_setup_all_rx_resources(adapter);
2281                 if (err)
2282                         goto reset_err;
2283
2284                 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
2285                         err = iavf_request_traffic_irqs(adapter, netdev->name);
2286                         if (err)
2287                                 goto reset_err;
2288
2289                         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
2290                 }
2291
2292                 iavf_configure(adapter);
2293
2294                 iavf_up_complete(adapter);
2295
2296                 iavf_irq_enable(adapter, true);
2297         } else {
2298                 adapter->state = __IAVF_DOWN;
2299                 wake_up(&adapter->down_waitqueue);
2300         }
2301         mutex_unlock(&adapter->client_lock);
2302         mutex_unlock(&adapter->crit_lock);
2303
2304         return;
2305 reset_err:
2306         mutex_unlock(&adapter->client_lock);
2307         mutex_unlock(&adapter->crit_lock);
2308         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
2309         iavf_close(netdev);
2310 }
2311
2312 /**
2313  * iavf_adminq_task - worker thread to clean the admin queue
2314  * @work: pointer to work_struct containing our data
2315  **/
2316 static void iavf_adminq_task(struct work_struct *work)
2317 {
2318         struct iavf_adapter *adapter =
2319                 container_of(work, struct iavf_adapter, adminq_task);
2320         struct iavf_hw *hw = &adapter->hw;
2321         struct iavf_arq_event_info event;
2322         enum virtchnl_ops v_op;
2323         enum iavf_status ret, v_ret;
2324         u32 val, oldval;
2325         u16 pending;
2326
2327         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2328                 goto out;
2329
2330         event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
2331         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
2332         if (!event.msg_buf)
2333                 goto out;
2334
2335         if (iavf_lock_timeout(&adapter->crit_lock, 200))
2336                 goto freedom;
2337         do {
2338                 ret = iavf_clean_arq_element(hw, &event, &pending);
2339                 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
2340                 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
2341
2342                 if (ret || !v_op)
2343                         break; /* No event to process or error cleaning ARQ */
2344
2345                 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
2346                                          event.msg_len);
2347                 if (pending != 0)
2348                         memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
2349         } while (pending);
2350         mutex_unlock(&adapter->crit_lock);
2351
2352         if ((adapter->flags &
2353              (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
2354             adapter->state == __IAVF_RESETTING)
2355                 goto freedom;
2356
2357         /* check for error indications */
2358         val = rd32(hw, hw->aq.arq.len);
2359         if (val == 0xdeadbeef) /* indicates device in reset */
2360                 goto freedom;
2361         oldval = val;
2362         if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
2363                 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
2364                 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
2365         }
2366         if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
2367                 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
2368                 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
2369         }
2370         if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
2371                 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
2372                 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
2373         }
2374         if (oldval != val)
2375                 wr32(hw, hw->aq.arq.len, val);
2376
2377         val = rd32(hw, hw->aq.asq.len);
2378         oldval = val;
2379         if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
2380                 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
2381                 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
2382         }
2383         if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
2384                 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
2385                 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
2386         }
2387         if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
2388                 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
2389                 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
2390         }
2391         if (oldval != val)
2392                 wr32(hw, hw->aq.asq.len, val);
2393
2394 freedom:
2395         kfree(event.msg_buf);
2396 out:
2397         /* re-enable Admin queue interrupt cause */
2398         iavf_misc_irq_enable(adapter);
2399 }
2400
2401 /**
2402  * iavf_client_task - worker thread to perform client work
2403  * @work: pointer to work_struct containing our data
2404  *
2405  * This task handles client interactions. Because client calls can be
2406  * reentrant, we can't handle them in the watchdog.
2407  **/
2408 static void iavf_client_task(struct work_struct *work)
2409 {
2410         struct iavf_adapter *adapter =
2411                 container_of(work, struct iavf_adapter, client_task.work);
2412
2413         /* If we can't get the client bit, just give up. We'll be rescheduled
2414          * later.
2415          */
2416
2417         if (!mutex_trylock(&adapter->client_lock))
2418                 return;
2419
2420         if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
2421                 iavf_client_subtask(adapter);
2422                 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
2423                 goto out;
2424         }
2425         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
2426                 iavf_notify_client_l2_params(&adapter->vsi);
2427                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
2428                 goto out;
2429         }
2430         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
2431                 iavf_notify_client_close(&adapter->vsi, false);
2432                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
2433                 goto out;
2434         }
2435         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
2436                 iavf_notify_client_open(&adapter->vsi);
2437                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
2438         }
2439 out:
2440         mutex_unlock(&adapter->client_lock);
2441 }
2442
2443 /**
2444  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
2445  * @adapter: board private structure
2446  *
2447  * Free all transmit software resources
2448  **/
2449 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
2450 {
2451         int i;
2452
2453         if (!adapter->tx_rings)
2454                 return;
2455
2456         for (i = 0; i < adapter->num_active_queues; i++)
2457                 if (adapter->tx_rings[i].desc)
2458                         iavf_free_tx_resources(&adapter->tx_rings[i]);
2459 }
2460
2461 /**
2462  * iavf_setup_all_tx_resources - allocate all queues Tx resources
2463  * @adapter: board private structure
2464  *
2465  * If this function returns with an error, then it's possible one or
2466  * more of the rings is populated (while the rest are not).  It is the
2467  * callers duty to clean those orphaned rings.
2468  *
2469  * Return 0 on success, negative on failure
2470  **/
2471 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
2472 {
2473         int i, err = 0;
2474
2475         for (i = 0; i < adapter->num_active_queues; i++) {
2476                 adapter->tx_rings[i].count = adapter->tx_desc_count;
2477                 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
2478                 if (!err)
2479                         continue;
2480                 dev_err(&adapter->pdev->dev,
2481                         "Allocation for Tx Queue %u failed\n", i);
2482                 break;
2483         }
2484
2485         return err;
2486 }
2487
2488 /**
2489  * iavf_setup_all_rx_resources - allocate all queues Rx resources
2490  * @adapter: board private structure
2491  *
2492  * If this function returns with an error, then it's possible one or
2493  * more of the rings is populated (while the rest are not).  It is the
2494  * callers duty to clean those orphaned rings.
2495  *
2496  * Return 0 on success, negative on failure
2497  **/
2498 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
2499 {
2500         int i, err = 0;
2501
2502         for (i = 0; i < adapter->num_active_queues; i++) {
2503                 adapter->rx_rings[i].count = adapter->rx_desc_count;
2504                 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
2505                 if (!err)
2506                         continue;
2507                 dev_err(&adapter->pdev->dev,
2508                         "Allocation for Rx Queue %u failed\n", i);
2509                 break;
2510         }
2511         return err;
2512 }
2513
2514 /**
2515  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
2516  * @adapter: board private structure
2517  *
2518  * Free all receive software resources
2519  **/
2520 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
2521 {
2522         int i;
2523
2524         if (!adapter->rx_rings)
2525                 return;
2526
2527         for (i = 0; i < adapter->num_active_queues; i++)
2528                 if (adapter->rx_rings[i].desc)
2529                         iavf_free_rx_resources(&adapter->rx_rings[i]);
2530 }
2531
2532 /**
2533  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
2534  * @adapter: board private structure
2535  * @max_tx_rate: max Tx bw for a tc
2536  **/
2537 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
2538                                       u64 max_tx_rate)
2539 {
2540         int speed = 0, ret = 0;
2541
2542         if (ADV_LINK_SUPPORT(adapter)) {
2543                 if (adapter->link_speed_mbps < U32_MAX) {
2544                         speed = adapter->link_speed_mbps;
2545                         goto validate_bw;
2546                 } else {
2547                         dev_err(&adapter->pdev->dev, "Unknown link speed\n");
2548                         return -EINVAL;
2549                 }
2550         }
2551
2552         switch (adapter->link_speed) {
2553         case VIRTCHNL_LINK_SPEED_40GB:
2554                 speed = SPEED_40000;
2555                 break;
2556         case VIRTCHNL_LINK_SPEED_25GB:
2557                 speed = SPEED_25000;
2558                 break;
2559         case VIRTCHNL_LINK_SPEED_20GB:
2560                 speed = SPEED_20000;
2561                 break;
2562         case VIRTCHNL_LINK_SPEED_10GB:
2563                 speed = SPEED_10000;
2564                 break;
2565         case VIRTCHNL_LINK_SPEED_5GB:
2566                 speed = SPEED_5000;
2567                 break;
2568         case VIRTCHNL_LINK_SPEED_2_5GB:
2569                 speed = SPEED_2500;
2570                 break;
2571         case VIRTCHNL_LINK_SPEED_1GB:
2572                 speed = SPEED_1000;
2573                 break;
2574         case VIRTCHNL_LINK_SPEED_100MB:
2575                 speed = SPEED_100;
2576                 break;
2577         default:
2578                 break;
2579         }
2580
2581 validate_bw:
2582         if (max_tx_rate > speed) {
2583                 dev_err(&adapter->pdev->dev,
2584                         "Invalid tx rate specified\n");
2585                 ret = -EINVAL;
2586         }
2587
2588         return ret;
2589 }
2590
2591 /**
2592  * iavf_validate_ch_config - validate queue mapping info
2593  * @adapter: board private structure
2594  * @mqprio_qopt: queue parameters
2595  *
2596  * This function validates if the config provided by the user to
2597  * configure queue channels is valid or not. Returns 0 on a valid
2598  * config.
2599  **/
2600 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
2601                                    struct tc_mqprio_qopt_offload *mqprio_qopt)
2602 {
2603         u64 total_max_rate = 0;
2604         int i, num_qps = 0;
2605         u64 tx_rate = 0;
2606         int ret = 0;
2607
2608         if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
2609             mqprio_qopt->qopt.num_tc < 1)
2610                 return -EINVAL;
2611
2612         for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
2613                 if (!mqprio_qopt->qopt.count[i] ||
2614                     mqprio_qopt->qopt.offset[i] != num_qps)
2615                         return -EINVAL;
2616                 if (mqprio_qopt->min_rate[i]) {
2617                         dev_err(&adapter->pdev->dev,
2618                                 "Invalid min tx rate (greater than 0) specified\n");
2619                         return -EINVAL;
2620                 }
2621                 /*convert to Mbps */
2622                 tx_rate = div_u64(mqprio_qopt->max_rate[i],
2623                                   IAVF_MBPS_DIVISOR);
2624                 total_max_rate += tx_rate;
2625                 num_qps += mqprio_qopt->qopt.count[i];
2626         }
2627         if (num_qps > IAVF_MAX_REQ_QUEUES)
2628                 return -EINVAL;
2629
2630         ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
2631         return ret;
2632 }
2633
2634 /**
2635  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
2636  * @adapter: board private structure
2637  **/
2638 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
2639 {
2640         struct iavf_cloud_filter *cf, *cftmp;
2641
2642         spin_lock_bh(&adapter->cloud_filter_list_lock);
2643         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
2644                                  list) {
2645                 list_del(&cf->list);
2646                 kfree(cf);
2647                 adapter->num_cloud_filters--;
2648         }
2649         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2650 }
2651
2652 /**
2653  * __iavf_setup_tc - configure multiple traffic classes
2654  * @netdev: network interface device structure
2655  * @type_data: tc offload data
2656  *
2657  * This function processes the config information provided by the
2658  * user to configure traffic classes/queue channels and packages the
2659  * information to request the PF to setup traffic classes.
2660  *
2661  * Returns 0 on success.
2662  **/
2663 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
2664 {
2665         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
2666         struct iavf_adapter *adapter = netdev_priv(netdev);
2667         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2668         u8 num_tc = 0, total_qps = 0;
2669         int ret = 0, netdev_tc = 0;
2670         u64 max_tx_rate;
2671         u16 mode;
2672         int i;
2673
2674         num_tc = mqprio_qopt->qopt.num_tc;
2675         mode = mqprio_qopt->mode;
2676
2677         /* delete queue_channel */
2678         if (!mqprio_qopt->qopt.hw) {
2679                 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
2680                         /* reset the tc configuration */
2681                         netdev_reset_tc(netdev);
2682                         adapter->num_tc = 0;
2683                         netif_tx_stop_all_queues(netdev);
2684                         netif_tx_disable(netdev);
2685                         iavf_del_all_cloud_filters(adapter);
2686                         adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
2687                         goto exit;
2688                 } else {
2689                         return -EINVAL;
2690                 }
2691         }
2692
2693         /* add queue channel */
2694         if (mode == TC_MQPRIO_MODE_CHANNEL) {
2695                 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
2696                         dev_err(&adapter->pdev->dev, "ADq not supported\n");
2697                         return -EOPNOTSUPP;
2698                 }
2699                 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
2700                         dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
2701                         return -EINVAL;
2702                 }
2703
2704                 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
2705                 if (ret)
2706                         return ret;
2707                 /* Return if same TC config is requested */
2708                 if (adapter->num_tc == num_tc)
2709                         return 0;
2710                 adapter->num_tc = num_tc;
2711
2712                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2713                         if (i < num_tc) {
2714                                 adapter->ch_config.ch_info[i].count =
2715                                         mqprio_qopt->qopt.count[i];
2716                                 adapter->ch_config.ch_info[i].offset =
2717                                         mqprio_qopt->qopt.offset[i];
2718                                 total_qps += mqprio_qopt->qopt.count[i];
2719                                 max_tx_rate = mqprio_qopt->max_rate[i];
2720                                 /* convert to Mbps */
2721                                 max_tx_rate = div_u64(max_tx_rate,
2722                                                       IAVF_MBPS_DIVISOR);
2723                                 adapter->ch_config.ch_info[i].max_tx_rate =
2724                                         max_tx_rate;
2725                         } else {
2726                                 adapter->ch_config.ch_info[i].count = 1;
2727                                 adapter->ch_config.ch_info[i].offset = 0;
2728                         }
2729                 }
2730                 adapter->ch_config.total_qps = total_qps;
2731                 netif_tx_stop_all_queues(netdev);
2732                 netif_tx_disable(netdev);
2733                 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
2734                 netdev_reset_tc(netdev);
2735                 /* Report the tc mapping up the stack */
2736                 netdev_set_num_tc(adapter->netdev, num_tc);
2737                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2738                         u16 qcount = mqprio_qopt->qopt.count[i];
2739                         u16 qoffset = mqprio_qopt->qopt.offset[i];
2740
2741                         if (i < num_tc)
2742                                 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
2743                                                     qoffset);
2744                 }
2745         }
2746 exit:
2747         return ret;
2748 }
2749
2750 /**
2751  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
2752  * @adapter: board private structure
2753  * @f: pointer to struct flow_cls_offload
2754  * @filter: pointer to cloud filter structure
2755  */
2756 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
2757                                  struct flow_cls_offload *f,
2758                                  struct iavf_cloud_filter *filter)
2759 {
2760         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2761         struct flow_dissector *dissector = rule->match.dissector;
2762         u16 n_proto_mask = 0;
2763         u16 n_proto_key = 0;
2764         u8 field_flags = 0;
2765         u16 addr_type = 0;
2766         u16 n_proto = 0;
2767         int i = 0;
2768         struct virtchnl_filter *vf = &filter->f;
2769
2770         if (dissector->used_keys &
2771             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2772               BIT(FLOW_DISSECTOR_KEY_BASIC) |
2773               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2774               BIT(FLOW_DISSECTOR_KEY_VLAN) |
2775               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
2776               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
2777               BIT(FLOW_DISSECTOR_KEY_PORTS) |
2778               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
2779                 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
2780                         dissector->used_keys);
2781                 return -EOPNOTSUPP;
2782         }
2783
2784         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
2785                 struct flow_match_enc_keyid match;
2786
2787                 flow_rule_match_enc_keyid(rule, &match);
2788                 if (match.mask->keyid != 0)
2789                         field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
2790         }
2791
2792         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2793                 struct flow_match_basic match;
2794
2795                 flow_rule_match_basic(rule, &match);
2796                 n_proto_key = ntohs(match.key->n_proto);
2797                 n_proto_mask = ntohs(match.mask->n_proto);
2798
2799                 if (n_proto_key == ETH_P_ALL) {
2800                         n_proto_key = 0;
2801                         n_proto_mask = 0;
2802                 }
2803                 n_proto = n_proto_key & n_proto_mask;
2804                 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
2805                         return -EINVAL;
2806                 if (n_proto == ETH_P_IPV6) {
2807                         /* specify flow type as TCP IPv6 */
2808                         vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
2809                 }
2810
2811                 if (match.key->ip_proto != IPPROTO_TCP) {
2812                         dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
2813                         return -EINVAL;
2814                 }
2815         }
2816
2817         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2818                 struct flow_match_eth_addrs match;
2819
2820                 flow_rule_match_eth_addrs(rule, &match);
2821
2822                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
2823                 if (!is_zero_ether_addr(match.mask->dst)) {
2824                         if (is_broadcast_ether_addr(match.mask->dst)) {
2825                                 field_flags |= IAVF_CLOUD_FIELD_OMAC;
2826                         } else {
2827                                 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
2828                                         match.mask->dst);
2829                                 return IAVF_ERR_CONFIG;
2830                         }
2831                 }
2832
2833                 if (!is_zero_ether_addr(match.mask->src)) {
2834                         if (is_broadcast_ether_addr(match.mask->src)) {
2835                                 field_flags |= IAVF_CLOUD_FIELD_IMAC;
2836                         } else {
2837                                 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
2838                                         match.mask->src);
2839                                 return IAVF_ERR_CONFIG;
2840                         }
2841                 }
2842
2843                 if (!is_zero_ether_addr(match.key->dst))
2844                         if (is_valid_ether_addr(match.key->dst) ||
2845                             is_multicast_ether_addr(match.key->dst)) {
2846                                 /* set the mask if a valid dst_mac address */
2847                                 for (i = 0; i < ETH_ALEN; i++)
2848                                         vf->mask.tcp_spec.dst_mac[i] |= 0xff;
2849                                 ether_addr_copy(vf->data.tcp_spec.dst_mac,
2850                                                 match.key->dst);
2851                         }
2852
2853                 if (!is_zero_ether_addr(match.key->src))
2854                         if (is_valid_ether_addr(match.key->src) ||
2855                             is_multicast_ether_addr(match.key->src)) {
2856                                 /* set the mask if a valid dst_mac address */
2857                                 for (i = 0; i < ETH_ALEN; i++)
2858                                         vf->mask.tcp_spec.src_mac[i] |= 0xff;
2859                                 ether_addr_copy(vf->data.tcp_spec.src_mac,
2860                                                 match.key->src);
2861                 }
2862         }
2863
2864         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2865                 struct flow_match_vlan match;
2866
2867                 flow_rule_match_vlan(rule, &match);
2868                 if (match.mask->vlan_id) {
2869                         if (match.mask->vlan_id == VLAN_VID_MASK) {
2870                                 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
2871                         } else {
2872                                 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
2873                                         match.mask->vlan_id);
2874                                 return IAVF_ERR_CONFIG;
2875                         }
2876                 }
2877                 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
2878                 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
2879         }
2880
2881         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
2882                 struct flow_match_control match;
2883
2884                 flow_rule_match_control(rule, &match);
2885                 addr_type = match.key->addr_type;
2886         }
2887
2888         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2889                 struct flow_match_ipv4_addrs match;
2890
2891                 flow_rule_match_ipv4_addrs(rule, &match);
2892                 if (match.mask->dst) {
2893                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
2894                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2895                         } else {
2896                                 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
2897                                         be32_to_cpu(match.mask->dst));
2898                                 return IAVF_ERR_CONFIG;
2899                         }
2900                 }
2901
2902                 if (match.mask->src) {
2903                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
2904                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2905                         } else {
2906                                 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
2907                                         be32_to_cpu(match.mask->dst));
2908                                 return IAVF_ERR_CONFIG;
2909                         }
2910                 }
2911
2912                 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
2913                         dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
2914                         return IAVF_ERR_CONFIG;
2915                 }
2916                 if (match.key->dst) {
2917                         vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
2918                         vf->data.tcp_spec.dst_ip[0] = match.key->dst;
2919                 }
2920                 if (match.key->src) {
2921                         vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
2922                         vf->data.tcp_spec.src_ip[0] = match.key->src;
2923                 }
2924         }
2925
2926         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2927                 struct flow_match_ipv6_addrs match;
2928
2929                 flow_rule_match_ipv6_addrs(rule, &match);
2930
2931                 /* validate mask, make sure it is not IPV6_ADDR_ANY */
2932                 if (ipv6_addr_any(&match.mask->dst)) {
2933                         dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
2934                                 IPV6_ADDR_ANY);
2935                         return IAVF_ERR_CONFIG;
2936                 }
2937
2938                 /* src and dest IPv6 address should not be LOOPBACK
2939                  * (0:0:0:0:0:0:0:1) which can be represented as ::1
2940                  */
2941                 if (ipv6_addr_loopback(&match.key->dst) ||
2942                     ipv6_addr_loopback(&match.key->src)) {
2943                         dev_err(&adapter->pdev->dev,
2944                                 "ipv6 addr should not be loopback\n");
2945                         return IAVF_ERR_CONFIG;
2946                 }
2947                 if (!ipv6_addr_any(&match.mask->dst) ||
2948                     !ipv6_addr_any(&match.mask->src))
2949                         field_flags |= IAVF_CLOUD_FIELD_IIP;
2950
2951                 for (i = 0; i < 4; i++)
2952                         vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
2953                 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
2954                        sizeof(vf->data.tcp_spec.dst_ip));
2955                 for (i = 0; i < 4; i++)
2956                         vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
2957                 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
2958                        sizeof(vf->data.tcp_spec.src_ip));
2959         }
2960         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
2961                 struct flow_match_ports match;
2962
2963                 flow_rule_match_ports(rule, &match);
2964                 if (match.mask->src) {
2965                         if (match.mask->src == cpu_to_be16(0xffff)) {
2966                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2967                         } else {
2968                                 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
2969                                         be16_to_cpu(match.mask->src));
2970                                 return IAVF_ERR_CONFIG;
2971                         }
2972                 }
2973
2974                 if (match.mask->dst) {
2975                         if (match.mask->dst == cpu_to_be16(0xffff)) {
2976                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2977                         } else {
2978                                 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
2979                                         be16_to_cpu(match.mask->dst));
2980                                 return IAVF_ERR_CONFIG;
2981                         }
2982                 }
2983                 if (match.key->dst) {
2984                         vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
2985                         vf->data.tcp_spec.dst_port = match.key->dst;
2986                 }
2987
2988                 if (match.key->src) {
2989                         vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
2990                         vf->data.tcp_spec.src_port = match.key->src;
2991                 }
2992         }
2993         vf->field_flags = field_flags;
2994
2995         return 0;
2996 }
2997
2998 /**
2999  * iavf_handle_tclass - Forward to a traffic class on the device
3000  * @adapter: board private structure
3001  * @tc: traffic class index on the device
3002  * @filter: pointer to cloud filter structure
3003  */
3004 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3005                               struct iavf_cloud_filter *filter)
3006 {
3007         if (tc == 0)
3008                 return 0;
3009         if (tc < adapter->num_tc) {
3010                 if (!filter->f.data.tcp_spec.dst_port) {
3011                         dev_err(&adapter->pdev->dev,
3012                                 "Specify destination port to redirect to traffic class other than TC0\n");
3013                         return -EINVAL;
3014                 }
3015         }
3016         /* redirect to a traffic class on the same device */
3017         filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3018         filter->f.action_meta = tc;
3019         return 0;
3020 }
3021
3022 /**
3023  * iavf_configure_clsflower - Add tc flower filters
3024  * @adapter: board private structure
3025  * @cls_flower: Pointer to struct flow_cls_offload
3026  */
3027 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3028                                     struct flow_cls_offload *cls_flower)
3029 {
3030         int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3031         struct iavf_cloud_filter *filter = NULL;
3032         int err = -EINVAL, count = 50;
3033
3034         if (tc < 0) {
3035                 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3036                 return -EINVAL;
3037         }
3038
3039         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
3040         if (!filter)
3041                 return -ENOMEM;
3042
3043         while (!mutex_trylock(&adapter->crit_lock)) {
3044                 if (--count == 0)
3045                         goto err;
3046                 udelay(1);
3047         }
3048
3049         filter->cookie = cls_flower->cookie;
3050
3051         /* set the mask to all zeroes to begin with */
3052         memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
3053         /* start out with flow type and eth type IPv4 to begin with */
3054         filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
3055         err = iavf_parse_cls_flower(adapter, cls_flower, filter);
3056         if (err < 0)
3057                 goto err;
3058
3059         err = iavf_handle_tclass(adapter, tc, filter);
3060         if (err < 0)
3061                 goto err;
3062
3063         /* add filter to the list */
3064         spin_lock_bh(&adapter->cloud_filter_list_lock);
3065         list_add_tail(&filter->list, &adapter->cloud_filter_list);
3066         adapter->num_cloud_filters++;
3067         filter->add = true;
3068         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3069         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3070 err:
3071         if (err)
3072                 kfree(filter);
3073
3074         mutex_unlock(&adapter->crit_lock);
3075         return err;
3076 }
3077
3078 /* iavf_find_cf - Find the cloud filter in the list
3079  * @adapter: Board private structure
3080  * @cookie: filter specific cookie
3081  *
3082  * Returns ptr to the filter object or NULL. Must be called while holding the
3083  * cloud_filter_list_lock.
3084  */
3085 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3086                                               unsigned long *cookie)
3087 {
3088         struct iavf_cloud_filter *filter = NULL;
3089
3090         if (!cookie)
3091                 return NULL;
3092
3093         list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3094                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3095                         return filter;
3096         }
3097         return NULL;
3098 }
3099
3100 /**
3101  * iavf_delete_clsflower - Remove tc flower filters
3102  * @adapter: board private structure
3103  * @cls_flower: Pointer to struct flow_cls_offload
3104  */
3105 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
3106                                  struct flow_cls_offload *cls_flower)
3107 {
3108         struct iavf_cloud_filter *filter = NULL;
3109         int err = 0;
3110
3111         spin_lock_bh(&adapter->cloud_filter_list_lock);
3112         filter = iavf_find_cf(adapter, &cls_flower->cookie);
3113         if (filter) {
3114                 filter->del = true;
3115                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
3116         } else {
3117                 err = -EINVAL;
3118         }
3119         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3120
3121         return err;
3122 }
3123
3124 /**
3125  * iavf_setup_tc_cls_flower - flower classifier offloads
3126  * @adapter: board private structure
3127  * @cls_flower: pointer to flow_cls_offload struct with flow info
3128  */
3129 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
3130                                     struct flow_cls_offload *cls_flower)
3131 {
3132         switch (cls_flower->command) {
3133         case FLOW_CLS_REPLACE:
3134                 return iavf_configure_clsflower(adapter, cls_flower);
3135         case FLOW_CLS_DESTROY:
3136                 return iavf_delete_clsflower(adapter, cls_flower);
3137         case FLOW_CLS_STATS:
3138                 return -EOPNOTSUPP;
3139         default:
3140                 return -EOPNOTSUPP;
3141         }
3142 }
3143
3144 /**
3145  * iavf_setup_tc_block_cb - block callback for tc
3146  * @type: type of offload
3147  * @type_data: offload data
3148  * @cb_priv:
3149  *
3150  * This function is the block callback for traffic classes
3151  **/
3152 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3153                                   void *cb_priv)
3154 {
3155         struct iavf_adapter *adapter = cb_priv;
3156
3157         if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
3158                 return -EOPNOTSUPP;
3159
3160         switch (type) {
3161         case TC_SETUP_CLSFLOWER:
3162                 return iavf_setup_tc_cls_flower(cb_priv, type_data);
3163         default:
3164                 return -EOPNOTSUPP;
3165         }
3166 }
3167
3168 static LIST_HEAD(iavf_block_cb_list);
3169
3170 /**
3171  * iavf_setup_tc - configure multiple traffic classes
3172  * @netdev: network interface device structure
3173  * @type: type of offload
3174  * @type_data: tc offload data
3175  *
3176  * This function is the callback to ndo_setup_tc in the
3177  * netdev_ops.
3178  *
3179  * Returns 0 on success
3180  **/
3181 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
3182                          void *type_data)
3183 {
3184         struct iavf_adapter *adapter = netdev_priv(netdev);
3185
3186         switch (type) {
3187         case TC_SETUP_QDISC_MQPRIO:
3188                 return __iavf_setup_tc(netdev, type_data);
3189         case TC_SETUP_BLOCK:
3190                 return flow_block_cb_setup_simple(type_data,
3191                                                   &iavf_block_cb_list,
3192                                                   iavf_setup_tc_block_cb,
3193                                                   adapter, adapter, true);
3194         default:
3195                 return -EOPNOTSUPP;
3196         }
3197 }
3198
3199 /**
3200  * iavf_open - Called when a network interface is made active
3201  * @netdev: network interface device structure
3202  *
3203  * Returns 0 on success, negative value on failure
3204  *
3205  * The open entry point is called when a network interface is made
3206  * active by the system (IFF_UP).  At this point all resources needed
3207  * for transmit and receive operations are allocated, the interrupt
3208  * handler is registered with the OS, the watchdog is started,
3209  * and the stack is notified that the interface is ready.
3210  **/
3211 static int iavf_open(struct net_device *netdev)
3212 {
3213         struct iavf_adapter *adapter = netdev_priv(netdev);
3214         int err;
3215
3216         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
3217                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
3218                 return -EIO;
3219         }
3220
3221         while (!mutex_trylock(&adapter->crit_lock))
3222                 usleep_range(500, 1000);
3223
3224         if (adapter->state != __IAVF_DOWN) {
3225                 err = -EBUSY;
3226                 goto err_unlock;
3227         }
3228
3229         /* allocate transmit descriptors */
3230         err = iavf_setup_all_tx_resources(adapter);
3231         if (err)
3232                 goto err_setup_tx;
3233
3234         /* allocate receive descriptors */
3235         err = iavf_setup_all_rx_resources(adapter);
3236         if (err)
3237                 goto err_setup_rx;
3238
3239         /* clear any pending interrupts, may auto mask */
3240         err = iavf_request_traffic_irqs(adapter, netdev->name);
3241         if (err)
3242                 goto err_req_irq;
3243
3244         spin_lock_bh(&adapter->mac_vlan_list_lock);
3245
3246         iavf_add_filter(adapter, adapter->hw.mac.addr);
3247
3248         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3249
3250         iavf_configure(adapter);
3251
3252         iavf_up_complete(adapter);
3253
3254         iavf_irq_enable(adapter, true);
3255
3256         mutex_unlock(&adapter->crit_lock);
3257
3258         return 0;
3259
3260 err_req_irq:
3261         iavf_down(adapter);
3262         iavf_free_traffic_irqs(adapter);
3263 err_setup_rx:
3264         iavf_free_all_rx_resources(adapter);
3265 err_setup_tx:
3266         iavf_free_all_tx_resources(adapter);
3267 err_unlock:
3268         mutex_unlock(&adapter->crit_lock);
3269
3270         return err;
3271 }
3272
3273 /**
3274  * iavf_close - Disables a network interface
3275  * @netdev: network interface device structure
3276  *
3277  * Returns 0, this is not allowed to fail
3278  *
3279  * The close entry point is called when an interface is de-activated
3280  * by the OS.  The hardware is still under the drivers control, but
3281  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
3282  * are freed, along with all transmit and receive resources.
3283  **/
3284 static int iavf_close(struct net_device *netdev)
3285 {
3286         struct iavf_adapter *adapter = netdev_priv(netdev);
3287         int status;
3288
3289         if (adapter->state <= __IAVF_DOWN_PENDING)
3290                 return 0;
3291
3292         while (!mutex_trylock(&adapter->crit_lock))
3293                 usleep_range(500, 1000);
3294
3295         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3296         if (CLIENT_ENABLED(adapter))
3297                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3298
3299         iavf_down(adapter);
3300         adapter->state = __IAVF_DOWN_PENDING;
3301         iavf_free_traffic_irqs(adapter);
3302
3303         mutex_unlock(&adapter->crit_lock);
3304
3305         /* We explicitly don't free resources here because the hardware is
3306          * still active and can DMA into memory. Resources are cleared in
3307          * iavf_virtchnl_completion() after we get confirmation from the PF
3308          * driver that the rings have been stopped.
3309          *
3310          * Also, we wait for state to transition to __IAVF_DOWN before
3311          * returning. State change occurs in iavf_virtchnl_completion() after
3312          * VF resources are released (which occurs after PF driver processes and
3313          * responds to admin queue commands).
3314          */
3315
3316         status = wait_event_timeout(adapter->down_waitqueue,
3317                                     adapter->state == __IAVF_DOWN,
3318                                     msecs_to_jiffies(500));
3319         if (!status)
3320                 netdev_warn(netdev, "Device resources not yet released\n");
3321         return 0;
3322 }
3323
3324 /**
3325  * iavf_change_mtu - Change the Maximum Transfer Unit
3326  * @netdev: network interface device structure
3327  * @new_mtu: new value for maximum frame size
3328  *
3329  * Returns 0 on success, negative on failure
3330  **/
3331 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
3332 {
3333         struct iavf_adapter *adapter = netdev_priv(netdev);
3334
3335         netdev->mtu = new_mtu;
3336         if (CLIENT_ENABLED(adapter)) {
3337                 iavf_notify_client_l2_params(&adapter->vsi);
3338                 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3339         }
3340         adapter->flags |= IAVF_FLAG_RESET_NEEDED;
3341         queue_work(iavf_wq, &adapter->reset_task);
3342
3343         return 0;
3344 }
3345
3346 /**
3347  * iavf_set_features - set the netdev feature flags
3348  * @netdev: ptr to the netdev being adjusted
3349  * @features: the feature set that the stack is suggesting
3350  * Note: expects to be called while under rtnl_lock()
3351  **/
3352 static int iavf_set_features(struct net_device *netdev,
3353                              netdev_features_t features)
3354 {
3355         struct iavf_adapter *adapter = netdev_priv(netdev);
3356
3357         /* Don't allow changing VLAN_RX flag when adapter is not capable
3358          * of VLAN offload
3359          */
3360         if (!VLAN_ALLOWED(adapter)) {
3361                 if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX)
3362                         return -EINVAL;
3363         } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
3364                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3365                         adapter->aq_required |=
3366                                 IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
3367                 else
3368                         adapter->aq_required |=
3369                                 IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
3370         }
3371
3372         return 0;
3373 }
3374
3375 /**
3376  * iavf_features_check - Validate encapsulated packet conforms to limits
3377  * @skb: skb buff
3378  * @dev: This physical port's netdev
3379  * @features: Offload features that the stack believes apply
3380  **/
3381 static netdev_features_t iavf_features_check(struct sk_buff *skb,
3382                                              struct net_device *dev,
3383                                              netdev_features_t features)
3384 {
3385         size_t len;
3386
3387         /* No point in doing any of this if neither checksum nor GSO are
3388          * being requested for this frame.  We can rule out both by just
3389          * checking for CHECKSUM_PARTIAL
3390          */
3391         if (skb->ip_summed != CHECKSUM_PARTIAL)
3392                 return features;
3393
3394         /* We cannot support GSO if the MSS is going to be less than
3395          * 64 bytes.  If it is then we need to drop support for GSO.
3396          */
3397         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
3398                 features &= ~NETIF_F_GSO_MASK;
3399
3400         /* MACLEN can support at most 63 words */
3401         len = skb_network_header(skb) - skb->data;
3402         if (len & ~(63 * 2))
3403                 goto out_err;
3404
3405         /* IPLEN and EIPLEN can support at most 127 dwords */
3406         len = skb_transport_header(skb) - skb_network_header(skb);
3407         if (len & ~(127 * 4))
3408                 goto out_err;
3409
3410         if (skb->encapsulation) {
3411                 /* L4TUNLEN can support 127 words */
3412                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
3413                 if (len & ~(127 * 2))
3414                         goto out_err;
3415
3416                 /* IPLEN can support at most 127 dwords */
3417                 len = skb_inner_transport_header(skb) -
3418                       skb_inner_network_header(skb);
3419                 if (len & ~(127 * 4))
3420                         goto out_err;
3421         }
3422
3423         /* No need to validate L4LEN as TCP is the only protocol with a
3424          * a flexible value and we support all possible values supported
3425          * by TCP, which is at most 15 dwords
3426          */
3427
3428         return features;
3429 out_err:
3430         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3431 }
3432
3433 /**
3434  * iavf_fix_features - fix up the netdev feature bits
3435  * @netdev: our net device
3436  * @features: desired feature bits
3437  *
3438  * Returns fixed-up features bits
3439  **/
3440 static netdev_features_t iavf_fix_features(struct net_device *netdev,
3441                                            netdev_features_t features)
3442 {
3443         struct iavf_adapter *adapter = netdev_priv(netdev);
3444
3445         if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
3446                 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3447                               NETIF_F_HW_VLAN_CTAG_RX |
3448                               NETIF_F_HW_VLAN_CTAG_FILTER);
3449
3450         return features;
3451 }
3452
3453 static const struct net_device_ops iavf_netdev_ops = {
3454         .ndo_open               = iavf_open,
3455         .ndo_stop               = iavf_close,
3456         .ndo_start_xmit         = iavf_xmit_frame,
3457         .ndo_set_rx_mode        = iavf_set_rx_mode,
3458         .ndo_validate_addr      = eth_validate_addr,
3459         .ndo_set_mac_address    = iavf_set_mac,
3460         .ndo_change_mtu         = iavf_change_mtu,
3461         .ndo_tx_timeout         = iavf_tx_timeout,
3462         .ndo_vlan_rx_add_vid    = iavf_vlan_rx_add_vid,
3463         .ndo_vlan_rx_kill_vid   = iavf_vlan_rx_kill_vid,
3464         .ndo_features_check     = iavf_features_check,
3465         .ndo_fix_features       = iavf_fix_features,
3466         .ndo_set_features       = iavf_set_features,
3467         .ndo_setup_tc           = iavf_setup_tc,
3468 };
3469
3470 /**
3471  * iavf_check_reset_complete - check that VF reset is complete
3472  * @hw: pointer to hw struct
3473  *
3474  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
3475  **/
3476 static int iavf_check_reset_complete(struct iavf_hw *hw)
3477 {
3478         u32 rstat;
3479         int i;
3480
3481         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3482                 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
3483                              IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3484                 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
3485                     (rstat == VIRTCHNL_VFR_COMPLETED))
3486                         return 0;
3487                 usleep_range(10, 20);
3488         }
3489         return -EBUSY;
3490 }
3491
3492 /**
3493  * iavf_process_config - Process the config information we got from the PF
3494  * @adapter: board private structure
3495  *
3496  * Verify that we have a valid config struct, and set up our netdev features
3497  * and our VSI struct.
3498  **/
3499 int iavf_process_config(struct iavf_adapter *adapter)
3500 {
3501         struct virtchnl_vf_resource *vfres = adapter->vf_res;
3502         int i, num_req_queues = adapter->num_req_queues;
3503         struct net_device *netdev = adapter->netdev;
3504         struct iavf_vsi *vsi = &adapter->vsi;
3505         netdev_features_t hw_enc_features;
3506         netdev_features_t hw_features;
3507
3508         /* got VF config message back from PF, now we can parse it */
3509         for (i = 0; i < vfres->num_vsis; i++) {
3510                 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
3511                         adapter->vsi_res = &vfres->vsi_res[i];
3512         }
3513         if (!adapter->vsi_res) {
3514                 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
3515                 return -ENODEV;
3516         }
3517
3518         if (num_req_queues &&
3519             num_req_queues > adapter->vsi_res->num_queue_pairs) {
3520                 /* Problem.  The PF gave us fewer queues than what we had
3521                  * negotiated in our request.  Need a reset to see if we can't
3522                  * get back to a working state.
3523                  */
3524                 dev_err(&adapter->pdev->dev,
3525                         "Requested %d queues, but PF only gave us %d.\n",
3526                         num_req_queues,
3527                         adapter->vsi_res->num_queue_pairs);
3528                 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
3529                 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
3530                 iavf_schedule_reset(adapter);
3531                 return -ENODEV;
3532         }
3533         adapter->num_req_queues = 0;
3534
3535         hw_enc_features = NETIF_F_SG                    |
3536                           NETIF_F_IP_CSUM               |
3537                           NETIF_F_IPV6_CSUM             |
3538                           NETIF_F_HIGHDMA               |
3539                           NETIF_F_SOFT_FEATURES |
3540                           NETIF_F_TSO                   |
3541                           NETIF_F_TSO_ECN               |
3542                           NETIF_F_TSO6                  |
3543                           NETIF_F_SCTP_CRC              |
3544                           NETIF_F_RXHASH                |
3545                           NETIF_F_RXCSUM                |
3546                           0;
3547
3548         /* advertise to stack only if offloads for encapsulated packets is
3549          * supported
3550          */
3551         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
3552                 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL       |
3553                                    NETIF_F_GSO_GRE              |
3554                                    NETIF_F_GSO_GRE_CSUM         |
3555                                    NETIF_F_GSO_IPXIP4           |
3556                                    NETIF_F_GSO_IPXIP6           |
3557                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
3558                                    NETIF_F_GSO_PARTIAL          |
3559                                    0;
3560
3561                 if (!(vfres->vf_cap_flags &
3562                       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
3563                         netdev->gso_partial_features |=
3564                                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3565
3566                 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3567                 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
3568                 netdev->hw_enc_features |= hw_enc_features;
3569         }
3570         /* record features VLANs can make use of */
3571         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
3572
3573         /* Write features and hw_features separately to avoid polluting
3574          * with, or dropping, features that are set when we registered.
3575          */
3576         hw_features = hw_enc_features;
3577
3578         /* Enable VLAN features if supported */
3579         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3580                 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
3581                                 NETIF_F_HW_VLAN_CTAG_RX);
3582         /* Enable cloud filter if ADQ is supported */
3583         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
3584                 hw_features |= NETIF_F_HW_TC;
3585         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
3586                 hw_features |= NETIF_F_GSO_UDP_L4;
3587
3588         netdev->hw_features |= hw_features;
3589
3590         netdev->features |= hw_features;
3591
3592         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3593                 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3594
3595         netdev->priv_flags |= IFF_UNICAST_FLT;
3596
3597         /* Do not turn on offloads when they are requested to be turned off.
3598          * TSO needs minimum 576 bytes to work correctly.
3599          */
3600         if (netdev->wanted_features) {
3601                 if (!(netdev->wanted_features & NETIF_F_TSO) ||
3602                     netdev->mtu < 576)
3603                         netdev->features &= ~NETIF_F_TSO;
3604                 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
3605                     netdev->mtu < 576)
3606                         netdev->features &= ~NETIF_F_TSO6;
3607                 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
3608                         netdev->features &= ~NETIF_F_TSO_ECN;
3609                 if (!(netdev->wanted_features & NETIF_F_GRO))
3610                         netdev->features &= ~NETIF_F_GRO;
3611                 if (!(netdev->wanted_features & NETIF_F_GSO))
3612                         netdev->features &= ~NETIF_F_GSO;
3613         }
3614
3615         adapter->vsi.id = adapter->vsi_res->vsi_id;
3616
3617         adapter->vsi.back = adapter;
3618         adapter->vsi.base_vector = 1;
3619         adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK;
3620         vsi->netdev = adapter->netdev;
3621         vsi->qs_handle = adapter->vsi_res->qset_handle;
3622         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
3623                 adapter->rss_key_size = vfres->rss_key_size;
3624                 adapter->rss_lut_size = vfres->rss_lut_size;
3625         } else {
3626                 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
3627                 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
3628         }
3629
3630         return 0;
3631 }
3632
3633 /**
3634  * iavf_init_task - worker thread to perform delayed initialization
3635  * @work: pointer to work_struct containing our data
3636  *
3637  * This task completes the work that was begun in probe. Due to the nature
3638  * of VF-PF communications, we may need to wait tens of milliseconds to get
3639  * responses back from the PF. Rather than busy-wait in probe and bog down the
3640  * whole system, we'll do it in a task so we can sleep.
3641  * This task only runs during driver init. Once we've established
3642  * communications with the PF driver and set up our netdev, the watchdog
3643  * takes over.
3644  **/
3645 static void iavf_init_task(struct work_struct *work)
3646 {
3647         struct iavf_adapter *adapter = container_of(work,
3648                                                     struct iavf_adapter,
3649                                                     init_task.work);
3650         struct iavf_hw *hw = &adapter->hw;
3651
3652         if (iavf_lock_timeout(&adapter->crit_lock, 5000)) {
3653                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3654                 return;
3655         }
3656         switch (adapter->state) {
3657         case __IAVF_STARTUP:
3658                 if (iavf_startup(adapter) < 0)
3659                         goto init_failed;
3660                 break;
3661         case __IAVF_INIT_VERSION_CHECK:
3662                 if (iavf_init_version_check(adapter) < 0)
3663                         goto init_failed;
3664                 break;
3665         case __IAVF_INIT_GET_RESOURCES:
3666                 if (iavf_init_get_resources(adapter) < 0)
3667                         goto init_failed;
3668                 goto out;
3669         default:
3670                 goto init_failed;
3671         }
3672
3673         queue_delayed_work(iavf_wq, &adapter->init_task,
3674                            msecs_to_jiffies(30));
3675         goto out;
3676 init_failed:
3677         if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
3678                 dev_err(&adapter->pdev->dev,
3679                         "Failed to communicate with PF; waiting before retry\n");
3680                 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
3681                 iavf_shutdown_adminq(hw);
3682                 adapter->state = __IAVF_STARTUP;
3683                 queue_delayed_work(iavf_wq, &adapter->init_task, HZ * 5);
3684                 goto out;
3685         }
3686         queue_delayed_work(iavf_wq, &adapter->init_task, HZ);
3687 out:
3688         mutex_unlock(&adapter->crit_lock);
3689 }
3690
3691 /**
3692  * iavf_shutdown - Shutdown the device in preparation for a reboot
3693  * @pdev: pci device structure
3694  **/
3695 static void iavf_shutdown(struct pci_dev *pdev)
3696 {
3697         struct net_device *netdev = pci_get_drvdata(pdev);
3698         struct iavf_adapter *adapter = netdev_priv(netdev);
3699
3700         netif_device_detach(netdev);
3701
3702         if (netif_running(netdev))
3703                 iavf_close(netdev);
3704
3705         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
3706                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3707         /* Prevent the watchdog from running. */
3708         adapter->state = __IAVF_REMOVE;
3709         adapter->aq_required = 0;
3710         mutex_unlock(&adapter->crit_lock);
3711
3712 #ifdef CONFIG_PM
3713         pci_save_state(pdev);
3714
3715 #endif
3716         pci_disable_device(pdev);
3717 }
3718
3719 /**
3720  * iavf_probe - Device Initialization Routine
3721  * @pdev: PCI device information struct
3722  * @ent: entry in iavf_pci_tbl
3723  *
3724  * Returns 0 on success, negative on failure
3725  *
3726  * iavf_probe initializes an adapter identified by a pci_dev structure.
3727  * The OS initialization, configuring of the adapter private structure,
3728  * and a hardware reset occur.
3729  **/
3730 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3731 {
3732         struct net_device *netdev;
3733         struct iavf_adapter *adapter = NULL;
3734         struct iavf_hw *hw = NULL;
3735         int err;
3736
3737         err = pci_enable_device(pdev);
3738         if (err)
3739                 return err;
3740
3741         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3742         if (err) {
3743                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3744                 if (err) {
3745                         dev_err(&pdev->dev,
3746                                 "DMA configuration failed: 0x%x\n", err);
3747                         goto err_dma;
3748                 }
3749         }
3750
3751         err = pci_request_regions(pdev, iavf_driver_name);
3752         if (err) {
3753                 dev_err(&pdev->dev,
3754                         "pci_request_regions failed 0x%x\n", err);
3755                 goto err_pci_reg;
3756         }
3757
3758         pci_enable_pcie_error_reporting(pdev);
3759
3760         pci_set_master(pdev);
3761
3762         netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
3763                                    IAVF_MAX_REQ_QUEUES);
3764         if (!netdev) {
3765                 err = -ENOMEM;
3766                 goto err_alloc_etherdev;
3767         }
3768
3769         SET_NETDEV_DEV(netdev, &pdev->dev);
3770
3771         pci_set_drvdata(pdev, netdev);
3772         adapter = netdev_priv(netdev);
3773
3774         adapter->netdev = netdev;
3775         adapter->pdev = pdev;
3776
3777         hw = &adapter->hw;
3778         hw->back = adapter;
3779
3780         adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
3781         adapter->state = __IAVF_STARTUP;
3782
3783         /* Call save state here because it relies on the adapter struct. */
3784         pci_save_state(pdev);
3785
3786         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3787                               pci_resource_len(pdev, 0));
3788         if (!hw->hw_addr) {
3789                 err = -EIO;
3790                 goto err_ioremap;
3791         }
3792         hw->vendor_id = pdev->vendor;
3793         hw->device_id = pdev->device;
3794         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3795         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3796         hw->subsystem_device_id = pdev->subsystem_device;
3797         hw->bus.device = PCI_SLOT(pdev->devfn);
3798         hw->bus.func = PCI_FUNC(pdev->devfn);
3799         hw->bus.bus_id = pdev->bus->number;
3800
3801         /* set up the locks for the AQ, do this only once in probe
3802          * and destroy them only once in remove
3803          */
3804         mutex_init(&adapter->crit_lock);
3805         mutex_init(&adapter->client_lock);
3806         mutex_init(&adapter->remove_lock);
3807         mutex_init(&hw->aq.asq_mutex);
3808         mutex_init(&hw->aq.arq_mutex);
3809
3810         spin_lock_init(&adapter->mac_vlan_list_lock);
3811         spin_lock_init(&adapter->cloud_filter_list_lock);
3812         spin_lock_init(&adapter->fdir_fltr_lock);
3813         spin_lock_init(&adapter->adv_rss_lock);
3814
3815         INIT_LIST_HEAD(&adapter->mac_filter_list);
3816         INIT_LIST_HEAD(&adapter->vlan_filter_list);
3817         INIT_LIST_HEAD(&adapter->cloud_filter_list);
3818         INIT_LIST_HEAD(&adapter->fdir_list_head);
3819         INIT_LIST_HEAD(&adapter->adv_rss_list_head);
3820
3821         INIT_WORK(&adapter->reset_task, iavf_reset_task);
3822         INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
3823         INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
3824         INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
3825         INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task);
3826         queue_delayed_work(iavf_wq, &adapter->init_task,
3827                            msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
3828
3829         /* Setup the wait queue for indicating transition to down status */
3830         init_waitqueue_head(&adapter->down_waitqueue);
3831
3832         return 0;
3833
3834 err_ioremap:
3835         free_netdev(netdev);
3836 err_alloc_etherdev:
3837         pci_disable_pcie_error_reporting(pdev);
3838         pci_release_regions(pdev);
3839 err_pci_reg:
3840 err_dma:
3841         pci_disable_device(pdev);
3842         return err;
3843 }
3844
3845 /**
3846  * iavf_suspend - Power management suspend routine
3847  * @dev_d: device info pointer
3848  *
3849  * Called when the system (VM) is entering sleep/suspend.
3850  **/
3851 static int __maybe_unused iavf_suspend(struct device *dev_d)
3852 {
3853         struct net_device *netdev = dev_get_drvdata(dev_d);
3854         struct iavf_adapter *adapter = netdev_priv(netdev);
3855
3856         netif_device_detach(netdev);
3857
3858         while (!mutex_trylock(&adapter->crit_lock))
3859                 usleep_range(500, 1000);
3860
3861         if (netif_running(netdev)) {
3862                 rtnl_lock();
3863                 iavf_down(adapter);
3864                 rtnl_unlock();
3865         }
3866         iavf_free_misc_irq(adapter);
3867         iavf_reset_interrupt_capability(adapter);
3868
3869         mutex_unlock(&adapter->crit_lock);
3870
3871         return 0;
3872 }
3873
3874 /**
3875  * iavf_resume - Power management resume routine
3876  * @dev_d: device info pointer
3877  *
3878  * Called when the system (VM) is resumed from sleep/suspend.
3879  **/
3880 static int __maybe_unused iavf_resume(struct device *dev_d)
3881 {
3882         struct pci_dev *pdev = to_pci_dev(dev_d);
3883         struct net_device *netdev = pci_get_drvdata(pdev);
3884         struct iavf_adapter *adapter = netdev_priv(netdev);
3885         u32 err;
3886
3887         pci_set_master(pdev);
3888
3889         rtnl_lock();
3890         err = iavf_set_interrupt_capability(adapter);
3891         if (err) {
3892                 rtnl_unlock();
3893                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
3894                 return err;
3895         }
3896         err = iavf_request_misc_irq(adapter);
3897         rtnl_unlock();
3898         if (err) {
3899                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
3900                 return err;
3901         }
3902
3903         queue_work(iavf_wq, &adapter->reset_task);
3904
3905         netif_device_attach(netdev);
3906
3907         return err;
3908 }
3909
3910 /**
3911  * iavf_remove - Device Removal Routine
3912  * @pdev: PCI device information struct
3913  *
3914  * iavf_remove is called by the PCI subsystem to alert the driver
3915  * that it should release a PCI device.  The could be caused by a
3916  * Hot-Plug event, or because the driver is going to be removed from
3917  * memory.
3918  **/
3919 static void iavf_remove(struct pci_dev *pdev)
3920 {
3921         struct net_device *netdev = pci_get_drvdata(pdev);
3922         struct iavf_adapter *adapter = netdev_priv(netdev);
3923         struct iavf_fdir_fltr *fdir, *fdirtmp;
3924         struct iavf_vlan_filter *vlf, *vlftmp;
3925         struct iavf_adv_rss *rss, *rsstmp;
3926         struct iavf_mac_filter *f, *ftmp;
3927         struct iavf_cloud_filter *cf, *cftmp;
3928         struct iavf_hw *hw = &adapter->hw;
3929         int err;
3930         /* Indicate we are in remove and not to run reset_task */
3931         mutex_lock(&adapter->remove_lock);
3932         cancel_delayed_work_sync(&adapter->init_task);
3933         cancel_work_sync(&adapter->reset_task);
3934         cancel_delayed_work_sync(&adapter->client_task);
3935         if (adapter->netdev_registered) {
3936                 unregister_netdev(netdev);
3937                 adapter->netdev_registered = false;
3938         }
3939         if (CLIENT_ALLOWED(adapter)) {
3940                 err = iavf_lan_del_device(adapter);
3941                 if (err)
3942                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
3943                                  err);
3944         }
3945
3946         iavf_request_reset(adapter);
3947         msleep(50);
3948         /* If the FW isn't responding, kick it once, but only once. */
3949         if (!iavf_asq_done(hw)) {
3950                 iavf_request_reset(adapter);
3951                 msleep(50);
3952         }
3953         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
3954                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3955
3956         /* Shut down all the garbage mashers on the detention level */
3957         adapter->state = __IAVF_REMOVE;
3958         adapter->aq_required = 0;
3959         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3960         iavf_free_all_tx_resources(adapter);
3961         iavf_free_all_rx_resources(adapter);
3962         iavf_misc_irq_disable(adapter);
3963         iavf_free_misc_irq(adapter);
3964         iavf_reset_interrupt_capability(adapter);
3965         iavf_free_q_vectors(adapter);
3966
3967         cancel_delayed_work_sync(&adapter->watchdog_task);
3968
3969         cancel_work_sync(&adapter->adminq_task);
3970
3971         iavf_free_rss(adapter);
3972
3973         if (hw->aq.asq.count)
3974                 iavf_shutdown_adminq(hw);
3975
3976         /* destroy the locks only once, here */
3977         mutex_destroy(&hw->aq.arq_mutex);
3978         mutex_destroy(&hw->aq.asq_mutex);
3979         mutex_destroy(&adapter->client_lock);
3980         mutex_unlock(&adapter->crit_lock);
3981         mutex_destroy(&adapter->crit_lock);
3982         mutex_unlock(&adapter->remove_lock);
3983         mutex_destroy(&adapter->remove_lock);
3984
3985         iounmap(hw->hw_addr);
3986         pci_release_regions(pdev);
3987         iavf_free_queues(adapter);
3988         kfree(adapter->vf_res);
3989         spin_lock_bh(&adapter->mac_vlan_list_lock);
3990         /* If we got removed before an up/down sequence, we've got a filter
3991          * hanging out there that we need to get rid of.
3992          */
3993         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3994                 list_del(&f->list);
3995                 kfree(f);
3996         }
3997         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
3998                                  list) {
3999                 list_del(&vlf->list);
4000                 kfree(vlf);
4001         }
4002
4003         spin_unlock_bh(&adapter->mac_vlan_list_lock);
4004
4005         spin_lock_bh(&adapter->cloud_filter_list_lock);
4006         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
4007                 list_del(&cf->list);
4008                 kfree(cf);
4009         }
4010         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4011
4012         spin_lock_bh(&adapter->fdir_fltr_lock);
4013         list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
4014                 list_del(&fdir->list);
4015                 kfree(fdir);
4016         }
4017         spin_unlock_bh(&adapter->fdir_fltr_lock);
4018
4019         spin_lock_bh(&adapter->adv_rss_lock);
4020         list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
4021                                  list) {
4022                 list_del(&rss->list);
4023                 kfree(rss);
4024         }
4025         spin_unlock_bh(&adapter->adv_rss_lock);
4026
4027         free_netdev(netdev);
4028
4029         pci_disable_pcie_error_reporting(pdev);
4030
4031         pci_disable_device(pdev);
4032 }
4033
4034 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
4035
4036 static struct pci_driver iavf_driver = {
4037         .name      = iavf_driver_name,
4038         .id_table  = iavf_pci_tbl,
4039         .probe     = iavf_probe,
4040         .remove    = iavf_remove,
4041         .driver.pm = &iavf_pm_ops,
4042         .shutdown  = iavf_shutdown,
4043 };
4044
4045 /**
4046  * iavf_init_module - Driver Registration Routine
4047  *
4048  * iavf_init_module is the first routine called when the driver is
4049  * loaded. All it does is register with the PCI subsystem.
4050  **/
4051 static int __init iavf_init_module(void)
4052 {
4053         int ret;
4054
4055         pr_info("iavf: %s\n", iavf_driver_string);
4056
4057         pr_info("%s\n", iavf_copyright);
4058
4059         iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
4060                                   iavf_driver_name);
4061         if (!iavf_wq) {
4062                 pr_err("%s: Failed to create workqueue\n", iavf_driver_name);
4063                 return -ENOMEM;
4064         }
4065         ret = pci_register_driver(&iavf_driver);
4066         return ret;
4067 }
4068
4069 module_init(iavf_init_module);
4070
4071 /**
4072  * iavf_exit_module - Driver Exit Cleanup Routine
4073  *
4074  * iavf_exit_module is called just before the driver is removed
4075  * from memory.
4076  **/
4077 static void __exit iavf_exit_module(void)
4078 {
4079         pci_unregister_driver(&iavf_driver);
4080         destroy_workqueue(iavf_wq);
4081 }
4082
4083 module_exit(iavf_exit_module);
4084
4085 /* iavf_main.c */