drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/ipv6.h>
  46 #include <linux/slab.h>
  47 #include <net/checksum.h>
  48 #include <net/ip6_checksum.h>
  49 #include <linux/ethtool.h>
  50 #include <linux/if.h>
  51 #include <linux/if_vlan.h>
  52 #include <linux/prefetch.h>
  53
  54 #include "ixgbevf.h"
  55
  56 const char ixgbevf_driver_name[] = "ixgbevf";
  57 static const char ixgbevf_driver_string[] =
  58         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  59
  60 #define DRV_VERSION "2.6.0-k"
  61 const char ixgbevf_driver_version[] = DRV_VERSION;
  62 static char ixgbevf_copyright[] =
  63         "Copyright (c) 2009 - 2012 Intel Corporation.";
  64
  65 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  66         [board_82599_vf] = &ixgbevf_82599_vf_info,
  67         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  68 };
  69
  70 /* ixgbevf_pci_tbl - PCI Device ID Table
  71  *
  72  * Wildcard entries (PCI_ANY_ID) should come last
  73  * Last entry must be all 0s
  74  *
  75  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  76  *   Class, Class Mask, private data (not used) }
  77  */
  78 static struct pci_device_id ixgbevf_pci_tbl[] = {
  79         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
  80         board_82599_vf},
  81         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
  82         board_X540_vf},
  83
  84         /* required last entry */
  85         {0, }
  86 };
  87 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  88
  89 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  90 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
  91 MODULE_LICENSE("GPL");
  92 MODULE_VERSION(DRV_VERSION);
  93
  94 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  95 static int debug = -1;
  96 module_param(debug, int, 0);
  97 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  98
  99 /* forward decls */
 100 static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector);
 101 static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx,
 102                                u32 itr_reg);
 103
 104 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
 105                                            struct ixgbevf_ring *rx_ring,
 106                                            u32 val)
 107 {
 108         /*
 109          * Force memory writes to complete before letting h/w
 110          * know there are new descriptors to fetch.  (Only
 111          * applicable for weak-ordered memory model archs,
 112          * such as IA-64).
 113          */
 114         wmb();
 115         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
 116 }
 117
 118 /*
 119  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 120  * @adapter: pointer to adapter struct
 121  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 122  * @queue: queue to map the corresponding interrupt to
 123  * @msix_vector: the vector to map to the corresponding queue
 124  *
 125  */
 126 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 127                              u8 queue, u8 msix_vector)
 128 {
 129         u32 ivar, index;
 130         struct ixgbe_hw *hw = &adapter->hw;
 131         if (direction == -1) {
 132                 /* other causes */
 133                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 134                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 135                 ivar &= ~0xFF;
 136                 ivar |= msix_vector;
 137                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 138         } else {
 139                 /* tx or rx causes */
 140                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 141                 index = ((16 * (queue & 1)) + (8 * direction));
 142                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 143                 ivar &= ~(0xFF << index);
 144                 ivar |= (msix_vector << index);
 145                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 146         }
 147 }
 148
 149 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_adapter *adapter,
 150                                                struct ixgbevf_tx_buffer
 151                                                *tx_buffer_info)
 152 {
 153         if (tx_buffer_info->dma) {
 154                 if (tx_buffer_info->mapped_as_page)
 155                         dma_unmap_page(&adapter->pdev->dev,
 156                                        tx_buffer_info->dma,
 157                                        tx_buffer_info->length,
 158                                        DMA_TO_DEVICE);
 159                 else
 160                         dma_unmap_single(&adapter->pdev->dev,
 161                                          tx_buffer_info->dma,
 162                                          tx_buffer_info->length,
 163                                          DMA_TO_DEVICE);
 164                 tx_buffer_info->dma = 0;
 165         }
 166         if (tx_buffer_info->skb) {
 167                 dev_kfree_skb_any(tx_buffer_info->skb);
 168                 tx_buffer_info->skb = NULL;
 169         }
 170         tx_buffer_info->time_stamp = 0;
 171         /* tx_buffer_info must be completely set up in the transmit path */
 172 }
 173
 174 #define IXGBE_MAX_TXD_PWR       14
 175 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 176
 177 /* Tx Descriptors needed, worst case */
 178 #define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
 179                          (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
 180 #ifdef MAX_SKB_FRAGS
 181 #define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
 182         MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1)      /* for context */
 183 #else
 184 #define DESC_NEEDED TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD)
 185 #endif
 186
 187 static void ixgbevf_tx_timeout(struct net_device *netdev);
 188
 189 /**
 190  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 191  * @adapter: board private structure
 192  * @tx_ring: tx ring to clean
 193  **/
 194 static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter,
 195                                  struct ixgbevf_ring *tx_ring)
 196 {
 197         struct net_device *netdev = adapter->netdev;
 198         struct ixgbe_hw *hw = &adapter->hw;
 199         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 200         struct ixgbevf_tx_buffer *tx_buffer_info;
 201         unsigned int i, eop, count = 0;
 202         unsigned int total_bytes = 0, total_packets = 0;
 203
 204         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 205                 return true;
 206
 207         i = tx_ring->next_to_clean;
 208         eop = tx_ring->tx_buffer_info[i].next_to_watch;
 209         eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
 210
 211         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
 212                (count < tx_ring->work_limit)) {
 213                 bool cleaned = false;
 214                 rmb(); /* read buffer_info after eop_desc */
 215                 /* eop could change between read and DD-check */
 216                 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
 217                         goto cont_loop;
 218                 for ( ; !cleaned; count++) {
 219                         struct sk_buff *skb;
 220                         tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
 221                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 222                         cleaned = (i == eop);
 223                         skb = tx_buffer_info->skb;
 224
 225                         if (cleaned && skb) {
 226                                 unsigned int segs, bytecount;
 227
 228                                 /* gso_segs is currently only valid for tcp */
 229                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
 230                                 /* multiply data chunks by size of headers */
 231                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
 232                                             skb->len;
 233                                 total_packets += segs;
 234                                 total_bytes += bytecount;
 235                         }
 236
 237                         ixgbevf_unmap_and_free_tx_resource(adapter,
 238                                                            tx_buffer_info);
 239
 240                         tx_desc->wb.status = 0;
 241
 242                         i++;
 243                         if (i == tx_ring->count)
 244                                 i = 0;
 245                 }
 246
 247 cont_loop:
 248                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
 249                 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
 250         }
 251
 252         tx_ring->next_to_clean = i;
 253
 254 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 255         if (unlikely(count && netif_carrier_ok(netdev) &&
 256                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 257                 /* Make sure that anybody stopping the queue after this
 258                  * sees the new next_to_clean.
 259                  */
 260                 smp_mb();
 261 #ifdef HAVE_TX_MQ
 262                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
 263                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 264                         netif_wake_subqueue(netdev, tx_ring->queue_index);
 265                         ++adapter->restart_queue;
 266                 }
 267 #else
 268                 if (netif_queue_stopped(netdev) &&
 269                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 270                         netif_wake_queue(netdev);
 271                         ++adapter->restart_queue;
 272                 }
 273 #endif
 274         }
 275
 276         /* re-arm the interrupt */
 277         if ((count >= tx_ring->work_limit) &&
 278             (!test_bit(__IXGBEVF_DOWN, &adapter->state))) {
 279                 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, tx_ring->v_idx);
 280         }
 281
 282         u64_stats_update_begin(&tx_ring->syncp);
 283         tx_ring->total_bytes += total_bytes;
 284         tx_ring->total_packets += total_packets;
 285         u64_stats_update_end(&tx_ring->syncp);
 286
 287         return count < tx_ring->work_limit;
 288 }
 289
 290 /**
 291  * ixgbevf_receive_skb - Send a completed packet up the stack
 292  * @q_vector: structure containing interrupt and ring information
 293  * @skb: packet to send up
 294  * @status: hardware indication of status of receive
 295  * @rx_ring: rx descriptor ring (for a specific queue) to setup
 296  * @rx_desc: rx descriptor
 297  **/
 298 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
 299                                 struct sk_buff *skb, u8 status,
 300                                 struct ixgbevf_ring *ring,
 301                                 union ixgbe_adv_rx_desc *rx_desc)
 302 {
 303         struct ixgbevf_adapter *adapter = q_vector->adapter;
 304         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
 305         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
 306
 307         if (is_vlan && test_bit(tag, adapter->active_vlans))
 308                 __vlan_hwaccel_put_tag(skb, tag);
 309
 310         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
 311                         napi_gro_receive(&q_vector->napi, skb);
 312         else
 313                         netif_rx(skb);
 314 }
 315
 316 /**
 317  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 318  * @adapter: address of board private structure
 319  * @status_err: hardware indication of status of receive
 320  * @skb: skb currently being received and modified
 321  **/
 322 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
 323                                        u32 status_err, struct sk_buff *skb)
 324 {
 325         skb_checksum_none_assert(skb);
 326
 327         /* Rx csum disabled */
 328         if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
 329                 return;
 330
 331         /* if IP and error */
 332         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
 333             (status_err & IXGBE_RXDADV_ERR_IPE)) {
 334                 adapter->hw_csum_rx_error++;
 335                 return;
 336         }
 337
 338         if (!(status_err & IXGBE_RXD_STAT_L4CS))
 339                 return;
 340
 341         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
 342                 adapter->hw_csum_rx_error++;
 343                 return;
 344         }
 345
 346         /* It must be a TCP or UDP packet with a valid checksum */
 347         skb->ip_summed = CHECKSUM_UNNECESSARY;
 348         adapter->hw_csum_rx_good++;
 349 }
 350
 351 /**
 352  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 353  * @adapter: address of board private structure
 354  **/
 355 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
 356                                      struct ixgbevf_ring *rx_ring,
 357                                      int cleaned_count)
 358 {
 359         struct pci_dev *pdev = adapter->pdev;
 360         union ixgbe_adv_rx_desc *rx_desc;
 361         struct ixgbevf_rx_buffer *bi;
 362         struct sk_buff *skb;
 363         unsigned int i;
 364         unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN;
 365
 366         i = rx_ring->next_to_use;
 367         bi = &rx_ring->rx_buffer_info[i];
 368
 369         while (cleaned_count--) {
 370                 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i);
 371
 372                 if (!bi->page_dma &&
 373                     (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) {
 374                         if (!bi->page) {
 375                                 bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
 376                                 if (!bi->page) {
 377                                         adapter->alloc_rx_page_failed++;
 378                                         goto no_buffers;
 379                                 }
 380                                 bi->page_offset = 0;
 381                         } else {
 382                                 /* use a half page if we're re-using */
 383                                 bi->page_offset ^= (PAGE_SIZE / 2);
 384                         }
 385
 386                         bi->page_dma = dma_map_page(&pdev->dev, bi->page,
 387                                                     bi->page_offset,
 388                                                     (PAGE_SIZE / 2),
 389                                                     DMA_FROM_DEVICE);
 390                 }
 391
 392                 skb = bi->skb;
 393                 if (!skb) {
 394                         skb = netdev_alloc_skb(adapter->netdev,
 395                                                                bufsz);
 396
 397                         if (!skb) {
 398                                 adapter->alloc_rx_buff_failed++;
 399                                 goto no_buffers;
 400                         }
 401
 402                         /*
 403                          * Make buffer alignment 2 beyond a 16 byte boundary
 404                          * this will result in a 16 byte aligned IP header after
 405                          * the 14 byte MAC header is removed
 406                          */
 407                         skb_reserve(skb, NET_IP_ALIGN);
 408
 409                         bi->skb = skb;
 410                 }
 411                 if (!bi->dma) {
 412                         bi->dma = dma_map_single(&pdev->dev, skb->data,
 413                                                  rx_ring->rx_buf_len,
 414                                                  DMA_FROM_DEVICE);
 415                 }
 416                 /* Refresh the desc even if buffer_addrs didn't change because
 417                  * each write-back erases this info. */
 418                 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
 419                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
 420                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
 421                 } else {
 422                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
 423                 }
 424
 425                 i++;
 426                 if (i == rx_ring->count)
 427                         i = 0;
 428                 bi = &rx_ring->rx_buffer_info[i];
 429         }
 430
 431 no_buffers:
 432         if (rx_ring->next_to_use != i) {
 433                 rx_ring->next_to_use = i;
 434                 if (i-- == 0)
 435                         i = (rx_ring->count - 1);
 436
 437                 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
 438         }
 439 }
 440
 441 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 442                                              u64 qmask)
 443 {
 444         u32 mask;
 445         struct ixgbe_hw *hw = &adapter->hw;
 446
 447         mask = (qmask & 0xFFFFFFFF);
 448         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 449 }
 450
 451 static inline u16 ixgbevf_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc)
 452 {
 453         return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info;
 454 }
 455
 456 static inline u16 ixgbevf_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc)
 457 {
 458         return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
 459 }
 460
 461 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 462                                  struct ixgbevf_ring *rx_ring,
 463                                  int *work_done, int work_to_do)
 464 {
 465         struct ixgbevf_adapter *adapter = q_vector->adapter;
 466         struct pci_dev *pdev = adapter->pdev;
 467         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
 468         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
 469         struct sk_buff *skb;
 470         unsigned int i;
 471         u32 len, staterr;
 472         u16 hdr_info;
 473         bool cleaned = false;
 474         int cleaned_count = 0;
 475         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 476
 477         i = rx_ring->next_to_clean;
 478         rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i);
 479         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 480         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 481
 482         while (staterr & IXGBE_RXD_STAT_DD) {
 483                 u32 upper_len = 0;
 484                 if (*work_done >= work_to_do)
 485                         break;
 486                 (*work_done)++;
 487
 488                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 489                 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
 490                         hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc));
 491                         len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
 492                                IXGBE_RXDADV_HDRBUFLEN_SHIFT;
 493                         if (hdr_info & IXGBE_RXDADV_SPH)
 494                                 adapter->rx_hdr_split++;
 495                         if (len > IXGBEVF_RX_HDR_SIZE)
 496                                 len = IXGBEVF_RX_HDR_SIZE;
 497                         upper_len = le16_to_cpu(rx_desc->wb.upper.length);
 498                 } else {
 499                         len = le16_to_cpu(rx_desc->wb.upper.length);
 500                 }
 501                 cleaned = true;
 502                 skb = rx_buffer_info->skb;
 503                 prefetch(skb->data - NET_IP_ALIGN);
 504                 rx_buffer_info->skb = NULL;
 505
 506                 if (rx_buffer_info->dma) {
 507                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
 508                                          rx_ring->rx_buf_len,
 509                                          DMA_FROM_DEVICE);
 510                         rx_buffer_info->dma = 0;
 511                         skb_put(skb, len);
 512                 }
 513
 514                 if (upper_len) {
 515                         dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma,
 516                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
 517                         rx_buffer_info->page_dma = 0;
 518                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 519                                            rx_buffer_info->page,
 520                                            rx_buffer_info->page_offset,
 521                                            upper_len);
 522
 523                         if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) ||
 524                             (page_count(rx_buffer_info->page) != 1))
 525                                 rx_buffer_info->page = NULL;
 526                         else
 527                                 get_page(rx_buffer_info->page);
 528
 529                         skb->len += upper_len;
 530                         skb->data_len += upper_len;
 531                         skb->truesize += upper_len;
 532                 }
 533
 534                 i++;
 535                 if (i == rx_ring->count)
 536                         i = 0;
 537
 538                 next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i);
 539                 prefetch(next_rxd);
 540                 cleaned_count++;
 541
 542                 next_buffer = &rx_ring->rx_buffer_info[i];
 543
 544                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
 545                         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
 546                                 rx_buffer_info->skb = next_buffer->skb;
 547                                 rx_buffer_info->dma = next_buffer->dma;
 548                                 next_buffer->skb = skb;
 549                                 next_buffer->dma = 0;
 550                         } else {
 551                                 skb->next = next_buffer->skb;
 552                                 skb->next->prev = skb;
 553                         }
 554                         adapter->non_eop_descs++;
 555                         goto next_desc;
 556                 }
 557
 558                 /* ERR_MASK will only have valid bits if EOP set */
 559                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
 560                         dev_kfree_skb_irq(skb);
 561                         goto next_desc;
 562                 }
 563
 564                 ixgbevf_rx_checksum(adapter, staterr, skb);
 565
 566                 /* probably a little skewed due to removing CRC */
 567                 total_rx_bytes += skb->len;
 568                 total_rx_packets++;
 569
 570                 /*
 571                  * Work around issue of some types of VM to VM loop back
 572                  * packets not getting split correctly
 573                  */
 574                 if (staterr & IXGBE_RXD_STAT_LB) {
 575                         u32 header_fixup_len = skb_headlen(skb);
 576                         if (header_fixup_len < 14)
 577                                 skb_push(skb, header_fixup_len);
 578                 }
 579                 skb->protocol = eth_type_trans(skb, adapter->netdev);
 580
 581                 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
 582
 583 next_desc:
 584                 rx_desc->wb.upper.status_error = 0;
 585
 586                 /* return some buffers to hardware, one at a time is too slow */
 587                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 588                         ixgbevf_alloc_rx_buffers(adapter, rx_ring,
 589                                                  cleaned_count);
 590                         cleaned_count = 0;
 591                 }
 592
 593                 /* use prefetched values */
 594                 rx_desc = next_rxd;
 595                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
 596
 597                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 598         }
 599
 600         rx_ring->next_to_clean = i;
 601         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
 602
 603         if (cleaned_count)
 604                 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
 605
 606         u64_stats_update_begin(&rx_ring->syncp);
 607         rx_ring->total_packets += total_rx_packets;
 608         rx_ring->total_bytes += total_rx_bytes;
 609         u64_stats_update_end(&rx_ring->syncp);
 610
 611         return cleaned;
 612 }
 613
 614 /**
 615  * ixgbevf_clean_rxonly - msix (aka one shot) rx clean routine
 616  * @napi: napi struct with our devices info in it
 617  * @budget: amount of work driver is allowed to do this pass, in packets
 618  *
 619  * This function is optimized for cleaning one queue only on a single
 620  * q_vector!!!
 621  **/
 622 static int ixgbevf_clean_rxonly(struct napi_struct *napi, int budget)
 623 {
 624         struct ixgbevf_q_vector *q_vector =
 625                 container_of(napi, struct ixgbevf_q_vector, napi);
 626         struct ixgbevf_adapter *adapter = q_vector->adapter;
 627         struct ixgbevf_ring *rx_ring = NULL;
 628         int work_done = 0;
 629         long r_idx;
 630
 631         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
 632         rx_ring = &(adapter->rx_ring[r_idx]);
 633
 634         ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
 635
 636         /* If all Rx work done, exit the polling mode */
 637         if (work_done < budget) {
 638                 napi_complete(napi);
 639                 if (adapter->itr_setting & 1)
 640                         ixgbevf_set_itr_msix(q_vector);
 641                 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 642                         ixgbevf_irq_enable_queues(adapter, rx_ring->v_idx);
 643         }
 644
 645         return work_done;
 646 }
 647
 648 /**
 649  * ixgbevf_clean_rxonly_many - msix (aka one shot) rx clean routine
 650  * @napi: napi struct with our devices info in it
 651  * @budget: amount of work driver is allowed to do this pass, in packets
 652  *
 653  * This function will clean more than one rx queue associated with a
 654  * q_vector.
 655  **/
 656 static int ixgbevf_clean_rxonly_many(struct napi_struct *napi, int budget)
 657 {
 658         struct ixgbevf_q_vector *q_vector =
 659                 container_of(napi, struct ixgbevf_q_vector, napi);
 660         struct ixgbevf_adapter *adapter = q_vector->adapter;
 661         struct ixgbevf_ring *rx_ring = NULL;
 662         int work_done = 0, i;
 663         long r_idx;
 664         u64 enable_mask = 0;
 665
 666         /* attempt to distribute budget to each queue fairly, but don't allow
 667          * the budget to go below 1 because we'll exit polling */
 668         budget /= (q_vector->rxr_count ?: 1);
 669         budget = max(budget, 1);
 670         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
 671         for (i = 0; i < q_vector->rxr_count; i++) {
 672                 rx_ring = &(adapter->rx_ring[r_idx]);
 673                 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
 674                 enable_mask |= rx_ring->v_idx;
 675                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
 676                                       r_idx + 1);
 677         }
 678
 679 #ifndef HAVE_NETDEV_NAPI_LIST
 680         if (!netif_running(adapter->netdev))
 681                 work_done = 0;
 682
 683 #endif
 684         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
 685         rx_ring = &(adapter->rx_ring[r_idx]);
 686
 687         /* If all Rx work done, exit the polling mode */
 688         if (work_done < budget) {
 689                 napi_complete(napi);
 690                 if (adapter->itr_setting & 1)
 691                         ixgbevf_set_itr_msix(q_vector);
 692                 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 693                         ixgbevf_irq_enable_queues(adapter, enable_mask);
 694         }
 695
 696         return work_done;
 697 }
 698
 699
 700 /**
 701  * ixgbevf_configure_msix - Configure MSI-X hardware
 702  * @adapter: board private structure
 703  *
 704  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 705  * interrupts.
 706  **/
 707 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
 708 {
 709         struct ixgbevf_q_vector *q_vector;
 710         struct ixgbe_hw *hw = &adapter->hw;
 711         int i, j, q_vectors, v_idx, r_idx;
 712         u32 mask;
 713
 714         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 715
 716         /*
 717          * Populate the IVAR table and set the ITR values to the
 718          * corresponding register.
 719          */
 720         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
 721                 q_vector = adapter->q_vector[v_idx];
 722                 /* XXX for_each_set_bit(...) */
 723                 r_idx = find_first_bit(q_vector->rxr_idx,
 724                                        adapter->num_rx_queues);
 725
 726                 for (i = 0; i < q_vector->rxr_count; i++) {
 727                         j = adapter->rx_ring[r_idx].reg_idx;
 728                         ixgbevf_set_ivar(adapter, 0, j, v_idx);
 729                         r_idx = find_next_bit(q_vector->rxr_idx,
 730                                               adapter->num_rx_queues,
 731                                               r_idx + 1);
 732                 }
 733                 r_idx = find_first_bit(q_vector->txr_idx,
 734                                        adapter->num_tx_queues);
 735
 736                 for (i = 0; i < q_vector->txr_count; i++) {
 737                         j = adapter->tx_ring[r_idx].reg_idx;
 738                         ixgbevf_set_ivar(adapter, 1, j, v_idx);
 739                         r_idx = find_next_bit(q_vector->txr_idx,
 740                                               adapter->num_tx_queues,
 741                                               r_idx + 1);
 742                 }
 743
 744                 /* if this is a tx only vector halve the interrupt rate */
 745                 if (q_vector->txr_count && !q_vector->rxr_count)
 746                         q_vector->eitr = (adapter->eitr_param >> 1);
 747                 else if (q_vector->rxr_count)
 748                         /* rx only */
 749                         q_vector->eitr = adapter->eitr_param;
 750
 751                 ixgbevf_write_eitr(adapter, v_idx, q_vector->eitr);
 752         }
 753
 754         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
 755
 756         /* set up to autoclear timer, and the vectors */
 757         mask = IXGBE_EIMS_ENABLE_MASK;
 758         mask &= ~IXGBE_EIMS_OTHER;
 759         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
 760 }
 761
 762 enum latency_range {
 763         lowest_latency = 0,
 764         low_latency = 1,
 765         bulk_latency = 2,
 766         latency_invalid = 255
 767 };
 768
 769 /**
 770  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
 771  * @adapter: pointer to adapter
 772  * @eitr: eitr setting (ints per sec) to give last timeslice
 773  * @itr_setting: current throttle rate in ints/second
 774  * @packets: the number of packets during this measurement interval
 775  * @bytes: the number of bytes during this measurement interval
 776  *
 777  *      Stores a new ITR value based on packets and byte
 778  *      counts during the last interrupt.  The advantage of per interrupt
 779  *      computation is faster updates and more accurate ITR for the current
 780  *      traffic pattern.  Constants in this function were computed
 781  *      based on theoretical maximum wire speed and thresholds were set based
 782  *      on testing data as well as attempting to minimize response time
 783  *      while increasing bulk throughput.
 784  **/
 785 static u8 ixgbevf_update_itr(struct ixgbevf_adapter *adapter,
 786                              u32 eitr, u8 itr_setting,
 787                              int packets, int bytes)
 788 {
 789         unsigned int retval = itr_setting;
 790         u32 timepassed_us;
 791         u64 bytes_perint;
 792
 793         if (packets == 0)
 794                 goto update_itr_done;
 795
 796
 797         /* simple throttlerate management
 798          *    0-20MB/s lowest (100000 ints/s)
 799          *   20-100MB/s low   (20000 ints/s)
 800          *  100-1249MB/s bulk (8000 ints/s)
 801          */
 802         /* what was last interrupt timeslice? */
 803         timepassed_us = 1000000/eitr;
 804         bytes_perint = bytes / timepassed_us; /* bytes/usec */
 805
 806         switch (itr_setting) {
 807         case lowest_latency:
 808                 if (bytes_perint > adapter->eitr_low)
 809                         retval = low_latency;
 810                 break;
 811         case low_latency:
 812                 if (bytes_perint > adapter->eitr_high)
 813                         retval = bulk_latency;
 814                 else if (bytes_perint <= adapter->eitr_low)
 815                         retval = lowest_latency;
 816                 break;
 817         case bulk_latency:
 818                 if (bytes_perint <= adapter->eitr_high)
 819                         retval = low_latency;
 820                 break;
 821         }
 822
 823 update_itr_done:
 824         return retval;
 825 }
 826
 827 /**
 828  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 829  * @adapter: pointer to adapter struct
 830  * @v_idx: vector index into q_vector array
 831  * @itr_reg: new value to be written in *register* format, not ints/s
 832  *
 833  * This function is made to be called by ethtool and by the driver
 834  * when it needs to update VTEITR registers at runtime.  Hardware
 835  * specific quirks/differences are taken care of here.
 836  */
 837 static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx,
 838                                u32 itr_reg)
 839 {
 840         struct ixgbe_hw *hw = &adapter->hw;
 841
 842         itr_reg = EITR_INTS_PER_SEC_TO_REG(itr_reg);
 843
 844         /*
 845          * set the WDIS bit to not clear the timer bits and cause an
 846          * immediate assertion of the interrupt
 847          */
 848         itr_reg |= IXGBE_EITR_CNT_WDIS;
 849
 850         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 851 }
 852
 853 static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector)
 854 {
 855         struct ixgbevf_adapter *adapter = q_vector->adapter;
 856         u32 new_itr;
 857         u8 current_itr, ret_itr;
 858         int i, r_idx, v_idx = q_vector->v_idx;
 859         struct ixgbevf_ring *rx_ring, *tx_ring;
 860
 861         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
 862         for (i = 0; i < q_vector->txr_count; i++) {
 863                 tx_ring = &(adapter->tx_ring[r_idx]);
 864                 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr,
 865                                              q_vector->tx_itr,
 866                                              tx_ring->total_packets,
 867                                              tx_ring->total_bytes);
 868                 /* if the result for this queue would decrease interrupt
 869                  * rate for this vector then use that result */
 870                 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ?
 871                                     q_vector->tx_itr - 1 : ret_itr);
 872                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
 873                                       r_idx + 1);
 874         }
 875
 876         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
 877         for (i = 0; i < q_vector->rxr_count; i++) {
 878                 rx_ring = &(adapter->rx_ring[r_idx]);
 879                 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr,
 880                                              q_vector->rx_itr,
 881                                              rx_ring->total_packets,
 882                                              rx_ring->total_bytes);
 883                 /* if the result for this queue would decrease interrupt
 884                  * rate for this vector then use that result */
 885                 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ?
 886                                     q_vector->rx_itr - 1 : ret_itr);
 887                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
 888                                       r_idx + 1);
 889         }
 890
 891         current_itr = max(q_vector->rx_itr, q_vector->tx_itr);
 892
 893         switch (current_itr) {
 894         /* counts and packets in update_itr are dependent on these numbers */
 895         case lowest_latency:
 896                 new_itr = 100000;
 897                 break;
 898         case low_latency:
 899                 new_itr = 20000; /* aka hwitr = ~200 */
 900                 break;
 901         case bulk_latency:
 902         default:
 903                 new_itr = 8000;
 904                 break;
 905         }
 906
 907         if (new_itr != q_vector->eitr) {
 908                 u32 itr_reg;
 909
 910                 /* save the algorithm value here, not the smoothed one */
 911                 q_vector->eitr = new_itr;
 912                 /* do an exponential smoothing */
 913                 new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100);
 914                 itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr);
 915                 ixgbevf_write_eitr(adapter, v_idx, itr_reg);
 916         }
 917 }
 918
 919 static irqreturn_t ixgbevf_msix_mbx(int irq, void *data)
 920 {
 921         struct net_device *netdev = data;
 922         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
 923         struct ixgbe_hw *hw = &adapter->hw;
 924         u32 eicr;
 925         u32 msg;
 926         bool got_ack = false;
 927
 928         eicr = IXGBE_READ_REG(hw, IXGBE_VTEICS);
 929         IXGBE_WRITE_REG(hw, IXGBE_VTEICR, eicr);
 930
 931         if (!hw->mbx.ops.check_for_ack(hw))
 932                 got_ack = true;
 933
 934         if (!hw->mbx.ops.check_for_msg(hw)) {
 935                 hw->mbx.ops.read(hw, &msg, 1);
 936
 937                 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG)
 938                         mod_timer(&adapter->watchdog_timer,
 939                                   round_jiffies(jiffies + 1));
 940
 941                 if (msg & IXGBE_VT_MSGTYPE_NACK)
 942                         pr_warn("Last Request of type %2.2x to PF Nacked\n",
 943                                 msg & 0xFF);
 944                 /*
 945                  * Restore the PFSTS bit in case someone is polling for a
 946                  * return message from the PF
 947                  */
 948                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
 949         }
 950
 951         /*
 952          * checking for the ack clears the PFACK bit.  Place
 953          * it back in the v2p_mailbox cache so that anyone
 954          * polling for an ack will not miss it
 955          */
 956         if (got_ack)
 957                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
 958
 959         return IRQ_HANDLED;
 960 }
 961
 962 static irqreturn_t ixgbevf_msix_clean_tx(int irq, void *data)
 963 {
 964         struct ixgbevf_q_vector *q_vector = data;
 965         struct ixgbevf_adapter  *adapter = q_vector->adapter;
 966         struct ixgbevf_ring     *tx_ring;
 967         int i, r_idx;
 968
 969         if (!q_vector->txr_count)
 970                 return IRQ_HANDLED;
 971
 972         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
 973         for (i = 0; i < q_vector->txr_count; i++) {
 974                 tx_ring = &(adapter->tx_ring[r_idx]);
 975                 ixgbevf_clean_tx_irq(adapter, tx_ring);
 976                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
 977                                       r_idx + 1);
 978         }
 979
 980         if (adapter->itr_setting & 1)
 981                 ixgbevf_set_itr_msix(q_vector);
 982
 983         return IRQ_HANDLED;
 984 }
 985
 986 /**
 987  * ixgbevf_msix_clean_rx - single unshared vector rx clean (all queues)
 988  * @irq: unused
 989  * @data: pointer to our q_vector struct for this interrupt vector
 990  **/
 991 static irqreturn_t ixgbevf_msix_clean_rx(int irq, void *data)
 992 {
 993         struct ixgbevf_q_vector *q_vector = data;
 994         struct ixgbevf_adapter  *adapter = q_vector->adapter;
 995         struct ixgbe_hw *hw = &adapter->hw;
 996         struct ixgbevf_ring  *rx_ring;
 997         int r_idx;
 998
 999         if (!q_vector->rxr_count)
1000                 return IRQ_HANDLED;
1001
1002         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
1003         rx_ring = &(adapter->rx_ring[r_idx]);
1004         /* disable interrupts on this vector only */
1005         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, rx_ring->v_idx);
1006         napi_schedule(&q_vector->napi);
1007
1008
1009         return IRQ_HANDLED;
1010 }
1011
1012 static irqreturn_t ixgbevf_msix_clean_many(int irq, void *data)
1013 {
1014         ixgbevf_msix_clean_rx(irq, data);
1015         ixgbevf_msix_clean_tx(irq, data);
1016
1017         return IRQ_HANDLED;
1018 }
1019
1020 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
1021                                      int r_idx)
1022 {
1023         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1024
1025         set_bit(r_idx, q_vector->rxr_idx);
1026         q_vector->rxr_count++;
1027         a->rx_ring[r_idx].v_idx = 1 << v_idx;
1028 }
1029
1030 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
1031                                      int t_idx)
1032 {
1033         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1034
1035         set_bit(t_idx, q_vector->txr_idx);
1036         q_vector->txr_count++;
1037         a->tx_ring[t_idx].v_idx = 1 << v_idx;
1038 }
1039
1040 /**
1041  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
1042  * @adapter: board private structure to initialize
1043  *
1044  * This function maps descriptor rings to the queue-specific vectors
1045  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
1046  * one vector per ring/queue, but on a constrained vector budget, we
1047  * group the rings as "efficiently" as possible.  You would add new
1048  * mapping configurations in here.
1049  **/
1050 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
1051 {
1052         int q_vectors;
1053         int v_start = 0;
1054         int rxr_idx = 0, txr_idx = 0;
1055         int rxr_remaining = adapter->num_rx_queues;
1056         int txr_remaining = adapter->num_tx_queues;
1057         int i, j;
1058         int rqpv, tqpv;
1059         int err = 0;
1060
1061         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1062
1063         /*
1064          * The ideal configuration...
1065          * We have enough vectors to map one per queue.
1066          */
1067         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
1068                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
1069                         map_vector_to_rxq(adapter, v_start, rxr_idx);
1070
1071                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
1072                         map_vector_to_txq(adapter, v_start, txr_idx);
1073                 goto out;
1074         }
1075
1076         /*
1077          * If we don't have enough vectors for a 1-to-1
1078          * mapping, we'll have to group them so there are
1079          * multiple queues per vector.
1080          */
1081         /* Re-adjusting *qpv takes care of the remainder. */
1082         for (i = v_start; i < q_vectors; i++) {
1083                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
1084                 for (j = 0; j < rqpv; j++) {
1085                         map_vector_to_rxq(adapter, i, rxr_idx);
1086                         rxr_idx++;
1087                         rxr_remaining--;
1088                 }
1089         }
1090         for (i = v_start; i < q_vectors; i++) {
1091                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
1092                 for (j = 0; j < tqpv; j++) {
1093                         map_vector_to_txq(adapter, i, txr_idx);
1094                         txr_idx++;
1095                         txr_remaining--;
1096                 }
1097         }
1098
1099 out:
1100         return err;
1101 }
1102
1103 /**
1104  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1105  * @adapter: board private structure
1106  *
1107  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1108  * interrupts from the kernel.
1109  **/
1110 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1111 {
1112         struct net_device *netdev = adapter->netdev;
1113         irqreturn_t (*handler)(int, void *);
1114         int i, vector, q_vectors, err;
1115         int ri = 0, ti = 0;
1116
1117         /* Decrement for Other and TCP Timer vectors */
1118         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1119
1120 #define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count)          \
1121                                           ? &ixgbevf_msix_clean_many : \
1122                           (_v)->rxr_count ? &ixgbevf_msix_clean_rx   : \
1123                           (_v)->txr_count ? &ixgbevf_msix_clean_tx   : \
1124                           NULL)
1125         for (vector = 0; vector < q_vectors; vector++) {
1126                 handler = SET_HANDLER(adapter->q_vector[vector]);
1127
1128                 if (handler == &ixgbevf_msix_clean_rx) {
1129                         sprintf(adapter->name[vector], "%s-%s-%d",
1130                                 netdev->name, "rx", ri++);
1131                 } else if (handler == &ixgbevf_msix_clean_tx) {
1132                         sprintf(adapter->name[vector], "%s-%s-%d",
1133                                 netdev->name, "tx", ti++);
1134                 } else if (handler == &ixgbevf_msix_clean_many) {
1135                         sprintf(adapter->name[vector], "%s-%s-%d",
1136                                 netdev->name, "TxRx", vector);
1137                 } else {
1138                         /* skip this unused q_vector */
1139                         continue;
1140                 }
1141                 err = request_irq(adapter->msix_entries[vector].vector,
1142                                   handler, 0, adapter->name[vector],
1143                                   adapter->q_vector[vector]);
1144                 if (err) {
1145                         hw_dbg(&adapter->hw,
1146                                "request_irq failed for MSIX interrupt "
1147                                "Error: %d\n", err);
1148                         goto free_queue_irqs;
1149                 }
1150         }
1151
1152         sprintf(adapter->name[vector], "%s:mbx", netdev->name);
1153         err = request_irq(adapter->msix_entries[vector].vector,
1154                           &ixgbevf_msix_mbx, 0, adapter->name[vector], netdev);
1155         if (err) {
1156                 hw_dbg(&adapter->hw,
1157                        "request_irq for msix_mbx failed: %d\n", err);
1158                 goto free_queue_irqs;
1159         }
1160
1161         return 0;
1162
1163 free_queue_irqs:
1164         for (i = vector - 1; i >= 0; i--)
1165                 free_irq(adapter->msix_entries[--vector].vector,
1166                          &(adapter->q_vector[i]));
1167         pci_disable_msix(adapter->pdev);
1168         kfree(adapter->msix_entries);
1169         adapter->msix_entries = NULL;
1170         return err;
1171 }
1172
1173 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1174 {
1175         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1176
1177         for (i = 0; i < q_vectors; i++) {
1178                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1179                 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES);
1180                 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES);
1181                 q_vector->rxr_count = 0;
1182                 q_vector->txr_count = 0;
1183                 q_vector->eitr = adapter->eitr_param;
1184         }
1185 }
1186
1187 /**
1188  * ixgbevf_request_irq - initialize interrupts
1189  * @adapter: board private structure
1190  *
1191  * Attempts to configure interrupts using the best available
1192  * capabilities of the hardware and kernel.
1193  **/
1194 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1195 {
1196         int err = 0;
1197
1198         err = ixgbevf_request_msix_irqs(adapter);
1199
1200         if (err)
1201                 hw_dbg(&adapter->hw,
1202                        "request_irq failed, Error %d\n", err);
1203
1204         return err;
1205 }
1206
1207 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1208 {
1209         struct net_device *netdev = adapter->netdev;
1210         int i, q_vectors;
1211
1212         q_vectors = adapter->num_msix_vectors;
1213
1214         i = q_vectors - 1;
1215
1216         free_irq(adapter->msix_entries[i].vector, netdev);
1217         i--;
1218
1219         for (; i >= 0; i--) {
1220                 free_irq(adapter->msix_entries[i].vector,
1221                          adapter->q_vector[i]);
1222         }
1223
1224         ixgbevf_reset_q_vectors(adapter);
1225 }
1226
1227 /**
1228  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1229  * @adapter: board private structure
1230  **/
1231 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1232 {
1233         int i;
1234         struct ixgbe_hw *hw = &adapter->hw;
1235
1236         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1237
1238         IXGBE_WRITE_FLUSH(hw);
1239
1240         for (i = 0; i < adapter->num_msix_vectors; i++)
1241                 synchronize_irq(adapter->msix_entries[i].vector);
1242 }
1243
1244 /**
1245  * ixgbevf_irq_enable - Enable default interrupt generation settings
1246  * @adapter: board private structure
1247  **/
1248 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter,
1249                                       bool queues, bool flush)
1250 {
1251         struct ixgbe_hw *hw = &adapter->hw;
1252         u32 mask;
1253         u64 qmask;
1254
1255         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1256         qmask = ~0;
1257
1258         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1259
1260         if (queues)
1261                 ixgbevf_irq_enable_queues(adapter, qmask);
1262
1263         if (flush)
1264                 IXGBE_WRITE_FLUSH(hw);
1265 }
1266
1267 /**
1268  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1269  * @adapter: board private structure
1270  *
1271  * Configure the Tx unit of the MAC after a reset.
1272  **/
1273 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1274 {
1275         u64 tdba;
1276         struct ixgbe_hw *hw = &adapter->hw;
1277         u32 i, j, tdlen, txctrl;
1278
1279         /* Setup the HW Tx Head and Tail descriptor pointers */
1280         for (i = 0; i < adapter->num_tx_queues; i++) {
1281                 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1282                 j = ring->reg_idx;
1283                 tdba = ring->dma;
1284                 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1285                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1286                                 (tdba & DMA_BIT_MASK(32)));
1287                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1288                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1289                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1290                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1291                 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1292                 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1293                 /* Disable Tx Head Writeback RO bit, since this hoses
1294                  * bookkeeping if things aren't delivered in order.
1295                  */
1296                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1297                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1298                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1299         }
1300 }
1301
1302 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1303
1304 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1305 {
1306         struct ixgbevf_ring *rx_ring;
1307         struct ixgbe_hw *hw = &adapter->hw;
1308         u32 srrctl;
1309
1310         rx_ring = &adapter->rx_ring[index];
1311
1312         srrctl = IXGBE_SRRCTL_DROP_EN;
1313
1314         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
1315                 u16 bufsz = IXGBEVF_RXBUFFER_2048;
1316                 /* grow the amount we can receive on large page machines */
1317                 if (bufsz < (PAGE_SIZE / 2))
1318                         bufsz = (PAGE_SIZE / 2);
1319                 /* cap the bufsz at our largest descriptor size */
1320                 bufsz = min((u16)IXGBEVF_MAX_RXBUFFER, bufsz);
1321
1322                 srrctl |= bufsz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1323                 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1324                 srrctl |= ((IXGBEVF_RX_HDR_SIZE <<
1325                            IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
1326                            IXGBE_SRRCTL_BSIZEHDR_MASK);
1327         } else {
1328                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1329
1330                 if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE)
1331                         srrctl |= IXGBEVF_RXBUFFER_2048 >>
1332                                 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1333                 else
1334                         srrctl |= rx_ring->rx_buf_len >>
1335                                 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1336         }
1337         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1338 }
1339
1340 /**
1341  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1342  * @adapter: board private structure
1343  *
1344  * Configure the Rx unit of the MAC after a reset.
1345  **/
1346 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1347 {
1348         u64 rdba;
1349         struct ixgbe_hw *hw = &adapter->hw;
1350         struct net_device *netdev = adapter->netdev;
1351         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1352         int i, j;
1353         u32 rdlen;
1354         int rx_buf_len;
1355
1356         /* Decide whether to use packet split mode or not */
1357         if (netdev->mtu > ETH_DATA_LEN) {
1358                 if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE)
1359                         adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
1360                 else
1361                         adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
1362         } else {
1363                 if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE)
1364                         adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
1365                 else
1366                         adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
1367         }
1368
1369         /* Set the RX buffer length according to the mode */
1370         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
1371                 /* PSRTYPE must be initialized in 82599 */
1372                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
1373                         IXGBE_PSRTYPE_UDPHDR |
1374                         IXGBE_PSRTYPE_IPV4HDR |
1375                         IXGBE_PSRTYPE_IPV6HDR |
1376                         IXGBE_PSRTYPE_L2HDR;
1377                 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1378                 rx_buf_len = IXGBEVF_RX_HDR_SIZE;
1379         } else {
1380                 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1381                 if (netdev->mtu <= ETH_DATA_LEN)
1382                         rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1383                 else
1384                         rx_buf_len = ALIGN(max_frame, 1024);
1385         }
1386
1387         rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1388         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1389          * the Base and Length of the Rx Descriptor Ring */
1390         for (i = 0; i < adapter->num_rx_queues; i++) {
1391                 rdba = adapter->rx_ring[i].dma;
1392                 j = adapter->rx_ring[i].reg_idx;
1393                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1394                                 (rdba & DMA_BIT_MASK(32)));
1395                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1396                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1397                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1398                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1399                 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1400                 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1401                 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1402
1403                 ixgbevf_configure_srrctl(adapter, j);
1404         }
1405 }
1406
1407 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1408 {
1409         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1410         struct ixgbe_hw *hw = &adapter->hw;
1411
1412         /* add VID to filter table */
1413         if (hw->mac.ops.set_vfta)
1414                 hw->mac.ops.set_vfta(hw, vid, 0, true);
1415         set_bit(vid, adapter->active_vlans);
1416
1417         return 0;
1418 }
1419
1420 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1421 {
1422         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1423         struct ixgbe_hw *hw = &adapter->hw;
1424
1425         /* remove VID from filter table */
1426         if (hw->mac.ops.set_vfta)
1427                 hw->mac.ops.set_vfta(hw, vid, 0, false);
1428         clear_bit(vid, adapter->active_vlans);
1429
1430         return 0;
1431 }
1432
1433 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1434 {
1435         u16 vid;
1436
1437         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1438                 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
1439 }
1440
1441 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1442 {
1443         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1444         struct ixgbe_hw *hw = &adapter->hw;
1445         int count = 0;
1446
1447         if ((netdev_uc_count(netdev)) > 10) {
1448                 pr_err("Too many unicast filters - No Space\n");
1449                 return -ENOSPC;
1450         }
1451
1452         if (!netdev_uc_empty(netdev)) {
1453                 struct netdev_hw_addr *ha;
1454                 netdev_for_each_uc_addr(ha, netdev) {
1455                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1456                         udelay(200);
1457                 }
1458         } else {
1459                 /*
1460                  * If the list is empty then send message to PF driver to
1461                  * clear all macvlans on this VF.
1462                  */
1463                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1464         }
1465
1466         return count;
1467 }
1468
1469 /**
1470  * ixgbevf_set_rx_mode - Multicast set
1471  * @netdev: network interface device structure
1472  *
1473  * The set_rx_method entry point is called whenever the multicast address
1474  * list or the network interface flags are updated.  This routine is
1475  * responsible for configuring the hardware for proper multicast mode.
1476  **/
1477 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1478 {
1479         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1480         struct ixgbe_hw *hw = &adapter->hw;
1481
1482         /* reprogram multicast list */
1483         if (hw->mac.ops.update_mc_addr_list)
1484                 hw->mac.ops.update_mc_addr_list(hw, netdev);
1485
1486         ixgbevf_write_uc_addr_list(netdev);
1487 }
1488
1489 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1490 {
1491         int q_idx;
1492         struct ixgbevf_q_vector *q_vector;
1493         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1494
1495         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1496                 struct napi_struct *napi;
1497                 q_vector = adapter->q_vector[q_idx];
1498                 if (!q_vector->rxr_count)
1499                         continue;
1500                 napi = &q_vector->napi;
1501                 if (q_vector->rxr_count > 1)
1502                         napi->poll = &ixgbevf_clean_rxonly_many;
1503
1504                 napi_enable(napi);
1505         }
1506 }
1507
1508 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1509 {
1510         int q_idx;
1511         struct ixgbevf_q_vector *q_vector;
1512         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1513
1514         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1515                 q_vector = adapter->q_vector[q_idx];
1516                 if (!q_vector->rxr_count)
1517                         continue;
1518                 napi_disable(&q_vector->napi);
1519         }
1520 }
1521
1522 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1523 {
1524         struct net_device *netdev = adapter->netdev;
1525         int i;
1526
1527         ixgbevf_set_rx_mode(netdev);
1528
1529         ixgbevf_restore_vlan(adapter);
1530
1531         ixgbevf_configure_tx(adapter);
1532         ixgbevf_configure_rx(adapter);
1533         for (i = 0; i < adapter->num_rx_queues; i++) {
1534                 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1535                 ixgbevf_alloc_rx_buffers(adapter, ring, ring->count);
1536                 ring->next_to_use = ring->count - 1;
1537                 writel(ring->next_to_use, adapter->hw.hw_addr + ring->tail);
1538         }
1539 }
1540
1541 #define IXGBE_MAX_RX_DESC_POLL 10
1542 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1543                                                 int rxr)
1544 {
1545         struct ixgbe_hw *hw = &adapter->hw;
1546         int j = adapter->rx_ring[rxr].reg_idx;
1547         int k;
1548
1549         for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1550                 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1551                         break;
1552                 else
1553                         msleep(1);
1554         }
1555         if (k >= IXGBE_MAX_RX_DESC_POLL) {
1556                 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1557                        "not set within the polling period\n", rxr);
1558         }
1559
1560         ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1561                                 (adapter->rx_ring[rxr].count - 1));
1562 }
1563
1564 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1565 {
1566         /* Only save pre-reset stats if there are some */
1567         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1568                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1569                         adapter->stats.base_vfgprc;
1570                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1571                         adapter->stats.base_vfgptc;
1572                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1573                         adapter->stats.base_vfgorc;
1574                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1575                         adapter->stats.base_vfgotc;
1576                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1577                         adapter->stats.base_vfmprc;
1578         }
1579 }
1580
1581 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1582 {
1583         struct ixgbe_hw *hw = &adapter->hw;
1584
1585         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1586         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1587         adapter->stats.last_vfgorc |=
1588                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1589         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1590         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1591         adapter->stats.last_vfgotc |=
1592                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1593         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1594
1595         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1596         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1597         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1598         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1599         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1600 }
1601
1602 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1603 {
1604         struct net_device *netdev = adapter->netdev;
1605         struct ixgbe_hw *hw = &adapter->hw;
1606         int i, j = 0;
1607         int num_rx_rings = adapter->num_rx_queues;
1608         u32 txdctl, rxdctl;
1609         u32 msg[2];
1610
1611         for (i = 0; i < adapter->num_tx_queues; i++) {
1612                 j = adapter->tx_ring[i].reg_idx;
1613                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1614                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1615                 txdctl |= (8 << 16);
1616                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1617         }
1618
1619         for (i = 0; i < adapter->num_tx_queues; i++) {
1620                 j = adapter->tx_ring[i].reg_idx;
1621                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1622                 txdctl |= IXGBE_TXDCTL_ENABLE;
1623                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1624         }
1625
1626         for (i = 0; i < num_rx_rings; i++) {
1627                 j = adapter->rx_ring[i].reg_idx;
1628                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1629                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1630                 if (hw->mac.type == ixgbe_mac_X540_vf) {
1631                         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1632                         rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1633                                    IXGBE_RXDCTL_RLPML_EN);
1634                 }
1635                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1636                 ixgbevf_rx_desc_queue_enable(adapter, i);
1637         }
1638
1639         ixgbevf_configure_msix(adapter);
1640
1641         if (hw->mac.ops.set_rar) {
1642                 if (is_valid_ether_addr(hw->mac.addr))
1643                         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1644                 else
1645                         hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1646         }
1647
1648         msg[0] = IXGBE_VF_SET_LPE;
1649         msg[1] = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1650         hw->mbx.ops.write_posted(hw, msg, 2);
1651
1652         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1653         ixgbevf_napi_enable_all(adapter);
1654
1655         /* enable transmits */
1656         netif_tx_start_all_queues(netdev);
1657
1658         ixgbevf_save_reset_stats(adapter);
1659         ixgbevf_init_last_counter_stats(adapter);
1660
1661         /* bring the link up in the watchdog, this could race with our first
1662          * link up interrupt but shouldn't be a problem */
1663         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1664         adapter->link_check_timeout = jiffies;
1665         mod_timer(&adapter->watchdog_timer, jiffies);
1666 }
1667
1668 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1669 {
1670         struct ixgbe_hw *hw = &adapter->hw;
1671
1672         ixgbevf_configure(adapter);
1673
1674         ixgbevf_up_complete(adapter);
1675
1676         /* clear any pending interrupts, may auto mask */
1677         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1678
1679         ixgbevf_irq_enable(adapter, true, true);
1680 }
1681
1682 /**
1683  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1684  * @adapter: board private structure
1685  * @rx_ring: ring to free buffers from
1686  **/
1687 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1688                                   struct ixgbevf_ring *rx_ring)
1689 {
1690         struct pci_dev *pdev = adapter->pdev;
1691         unsigned long size;
1692         unsigned int i;
1693
1694         if (!rx_ring->rx_buffer_info)
1695                 return;
1696
1697         /* Free all the Rx ring sk_buffs */
1698         for (i = 0; i < rx_ring->count; i++) {
1699                 struct ixgbevf_rx_buffer *rx_buffer_info;
1700
1701                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1702                 if (rx_buffer_info->dma) {
1703                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1704                                          rx_ring->rx_buf_len,
1705                                          DMA_FROM_DEVICE);
1706                         rx_buffer_info->dma = 0;
1707                 }
1708                 if (rx_buffer_info->skb) {
1709                         struct sk_buff *skb = rx_buffer_info->skb;
1710                         rx_buffer_info->skb = NULL;
1711                         do {
1712                                 struct sk_buff *this = skb;
1713                                 skb = skb->prev;
1714                                 dev_kfree_skb(this);
1715                         } while (skb);
1716                 }
1717                 if (!rx_buffer_info->page)
1718                         continue;
1719                 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma,
1720                                PAGE_SIZE / 2, DMA_FROM_DEVICE);
1721                 rx_buffer_info->page_dma = 0;
1722                 put_page(rx_buffer_info->page);
1723                 rx_buffer_info->page = NULL;
1724                 rx_buffer_info->page_offset = 0;
1725         }
1726
1727         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1728         memset(rx_ring->rx_buffer_info, 0, size);
1729
1730         /* Zero out the descriptor ring */
1731         memset(rx_ring->desc, 0, rx_ring->size);
1732
1733         rx_ring->next_to_clean = 0;
1734         rx_ring->next_to_use = 0;
1735
1736         if (rx_ring->head)
1737                 writel(0, adapter->hw.hw_addr + rx_ring->head);
1738         if (rx_ring->tail)
1739                 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1740 }
1741
1742 /**
1743  * ixgbevf_clean_tx_ring - Free Tx Buffers
1744  * @adapter: board private structure
1745  * @tx_ring: ring to be cleaned
1746  **/
1747 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1748                                   struct ixgbevf_ring *tx_ring)
1749 {
1750         struct ixgbevf_tx_buffer *tx_buffer_info;
1751         unsigned long size;
1752         unsigned int i;
1753
1754         if (!tx_ring->tx_buffer_info)
1755                 return;
1756
1757         /* Free all the Tx ring sk_buffs */
1758
1759         for (i = 0; i < tx_ring->count; i++) {
1760                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1761                 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info);
1762         }
1763
1764         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1765         memset(tx_ring->tx_buffer_info, 0, size);
1766
1767         memset(tx_ring->desc, 0, tx_ring->size);
1768
1769         tx_ring->next_to_use = 0;
1770         tx_ring->next_to_clean = 0;
1771
1772         if (tx_ring->head)
1773                 writel(0, adapter->hw.hw_addr + tx_ring->head);
1774         if (tx_ring->tail)
1775                 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1776 }
1777
1778 /**
1779  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1780  * @adapter: board private structure
1781  **/
1782 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1783 {
1784         int i;
1785
1786         for (i = 0; i < adapter->num_rx_queues; i++)
1787                 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1788 }
1789
1790 /**
1791  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1792  * @adapter: board private structure
1793  **/
1794 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1795 {
1796         int i;
1797
1798         for (i = 0; i < adapter->num_tx_queues; i++)
1799                 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1800 }
1801
1802 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1803 {
1804         struct net_device *netdev = adapter->netdev;
1805         struct ixgbe_hw *hw = &adapter->hw;
1806         u32 txdctl;
1807         int i, j;
1808
1809         /* signal that we are down to the interrupt handler */
1810         set_bit(__IXGBEVF_DOWN, &adapter->state);
1811         /* disable receives */
1812
1813         netif_tx_disable(netdev);
1814
1815         msleep(10);
1816
1817         netif_tx_stop_all_queues(netdev);
1818
1819         ixgbevf_irq_disable(adapter);
1820
1821         ixgbevf_napi_disable_all(adapter);
1822
1823         del_timer_sync(&adapter->watchdog_timer);
1824         /* can't call flush scheduled work here because it can deadlock
1825          * if linkwatch_event tries to acquire the rtnl_lock which we are
1826          * holding */
1827         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1828                 msleep(1);
1829
1830         /* disable transmits in the hardware now that interrupts are off */
1831         for (i = 0; i < adapter->num_tx_queues; i++) {
1832                 j = adapter->tx_ring[i].reg_idx;
1833                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1834                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1835                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1836         }
1837
1838         netif_carrier_off(netdev);
1839
1840         if (!pci_channel_offline(adapter->pdev))
1841                 ixgbevf_reset(adapter);
1842
1843         ixgbevf_clean_all_tx_rings(adapter);
1844         ixgbevf_clean_all_rx_rings(adapter);
1845 }
1846
1847 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1848 {
1849         struct ixgbe_hw *hw = &adapter->hw;
1850
1851         WARN_ON(in_interrupt());
1852
1853         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1854                 msleep(1);
1855
1856         /*
1857          * Check if PF is up before re-init.  If not then skip until
1858          * later when the PF is up and ready to service requests from
1859          * the VF via mailbox.  If the VF is up and running then the
1860          * watchdog task will continue to schedule reset tasks until
1861          * the PF is up and running.
1862          */
1863         if (!hw->mac.ops.reset_hw(hw)) {
1864                 ixgbevf_down(adapter);
1865                 ixgbevf_up(adapter);
1866         }
1867
1868         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1869 }
1870
1871 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1872 {
1873         struct ixgbe_hw *hw = &adapter->hw;
1874         struct net_device *netdev = adapter->netdev;
1875
1876         if (hw->mac.ops.reset_hw(hw))
1877                 hw_dbg(hw, "PF still resetting\n");
1878         else
1879                 hw->mac.ops.init_hw(hw);
1880
1881         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1882                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1883                        netdev->addr_len);
1884                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1885                        netdev->addr_len);
1886         }
1887 }
1888
1889 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1890                                          int vectors)
1891 {
1892         int err, vector_threshold;
1893
1894         /* We'll want at least 3 (vector_threshold):
1895          * 1) TxQ[0] Cleanup
1896          * 2) RxQ[0] Cleanup
1897          * 3) Other (Link Status Change, etc.)
1898          */
1899         vector_threshold = MIN_MSIX_COUNT;
1900
1901         /* The more we get, the more we will assign to Tx/Rx Cleanup
1902          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1903          * Right now, we simply care about how many we'll get; we'll
1904          * set them up later while requesting irq's.
1905          */
1906         while (vectors >= vector_threshold) {
1907                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1908                                       vectors);
1909                 if (!err) /* Success in acquiring all requested vectors. */
1910                         break;
1911                 else if (err < 0)
1912                         vectors = 0; /* Nasty failure, quit now */
1913                 else /* err == number of vectors we should try again with */
1914                         vectors = err;
1915         }
1916
1917         if (vectors < vector_threshold) {
1918                 /* Can't allocate enough MSI-X interrupts?  Oh well.
1919                  * This just means we'll go with either a single MSI
1920                  * vector or fall back to legacy interrupts.
1921                  */
1922                 hw_dbg(&adapter->hw,
1923                        "Unable to allocate MSI-X interrupts\n");
1924                 kfree(adapter->msix_entries);
1925                 adapter->msix_entries = NULL;
1926         } else {
1927                 /*
1928                  * Adjust for only the vectors we'll use, which is minimum
1929                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1930                  * vectors we were allocated.
1931                  */
1932                 adapter->num_msix_vectors = vectors;
1933         }
1934 }
1935
1936 /*
1937  * ixgbevf_set_num_queues: Allocate queues for device, feature dependent
1938  * @adapter: board private structure to initialize
1939  *
1940  * This is the top level queue allocation routine.  The order here is very
1941  * important, starting with the "most" number of features turned on at once,
1942  * and ending with the smallest set of features.  This way large combinations
1943  * can be allocated if they're turned on, and smaller combinations are the
1944  * fallthrough conditions.
1945  *
1946  **/
1947 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1948 {
1949         /* Start with base case */
1950         adapter->num_rx_queues = 1;
1951         adapter->num_tx_queues = 1;
1952         adapter->num_rx_pools = adapter->num_rx_queues;
1953         adapter->num_rx_queues_per_pool = 1;
1954 }
1955
1956 /**
1957  * ixgbevf_alloc_queues - Allocate memory for all rings
1958  * @adapter: board private structure to initialize
1959  *
1960  * We allocate one ring per queue at run-time since we don't know the
1961  * number of queues at compile-time.  The polling_netdev array is
1962  * intended for Multiqueue, but should work fine with a single queue.
1963  **/
1964 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1965 {
1966         int i;
1967
1968         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1969                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1970         if (!adapter->tx_ring)
1971                 goto err_tx_ring_allocation;
1972
1973         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1974                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1975         if (!adapter->rx_ring)
1976                 goto err_rx_ring_allocation;
1977
1978         for (i = 0; i < adapter->num_tx_queues; i++) {
1979                 adapter->tx_ring[i].count = adapter->tx_ring_count;
1980                 adapter->tx_ring[i].queue_index = i;
1981                 adapter->tx_ring[i].reg_idx = i;
1982         }
1983
1984         for (i = 0; i < adapter->num_rx_queues; i++) {
1985                 adapter->rx_ring[i].count = adapter->rx_ring_count;
1986                 adapter->rx_ring[i].queue_index = i;
1987                 adapter->rx_ring[i].reg_idx = i;
1988         }
1989
1990         return 0;
1991
1992 err_rx_ring_allocation:
1993         kfree(adapter->tx_ring);
1994 err_tx_ring_allocation:
1995         return -ENOMEM;
1996 }
1997
1998 /**
1999  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2000  * @adapter: board private structure to initialize
2001  *
2002  * Attempt to configure the interrupts using the best available
2003  * capabilities of the hardware and the kernel.
2004  **/
2005 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2006 {
2007         int err = 0;
2008         int vector, v_budget;
2009
2010         /*
2011          * It's easy to be greedy for MSI-X vectors, but it really
2012          * doesn't do us much good if we have a lot more vectors
2013          * than CPU's.  So let's be conservative and only ask for
2014          * (roughly) twice the number of vectors as there are CPU's.
2015          */
2016         v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
2017                        (int)(num_online_cpus() * 2)) + NON_Q_VECTORS;
2018
2019         /* A failure in MSI-X entry allocation isn't fatal, but it does
2020          * mean we disable MSI-X capabilities of the adapter. */
2021         adapter->msix_entries = kcalloc(v_budget,
2022                                         sizeof(struct msix_entry), GFP_KERNEL);
2023         if (!adapter->msix_entries) {
2024                 err = -ENOMEM;
2025                 goto out;
2026         }
2027
2028         for (vector = 0; vector < v_budget; vector++)
2029                 adapter->msix_entries[vector].entry = vector;
2030
2031         ixgbevf_acquire_msix_vectors(adapter, v_budget);
2032
2033 out:
2034         return err;
2035 }
2036
2037 /**
2038  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2039  * @adapter: board private structure to initialize
2040  *
2041  * We allocate one q_vector per queue interrupt.  If allocation fails we
2042  * return -ENOMEM.
2043  **/
2044 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2045 {
2046         int q_idx, num_q_vectors;
2047         struct ixgbevf_q_vector *q_vector;
2048         int napi_vectors;
2049         int (*poll)(struct napi_struct *, int);
2050
2051         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2052         napi_vectors = adapter->num_rx_queues;
2053         poll = &ixgbevf_clean_rxonly;
2054
2055         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2056                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2057                 if (!q_vector)
2058                         goto err_out;
2059                 q_vector->adapter = adapter;
2060                 q_vector->v_idx = q_idx;
2061                 q_vector->eitr = adapter->eitr_param;
2062                 if (q_idx < napi_vectors)
2063                         netif_napi_add(adapter->netdev, &q_vector->napi,
2064                                        (*poll), 64);
2065                 adapter->q_vector[q_idx] = q_vector;
2066         }
2067
2068         return 0;
2069
2070 err_out:
2071         while (q_idx) {
2072                 q_idx--;
2073                 q_vector = adapter->q_vector[q_idx];
2074                 netif_napi_del(&q_vector->napi);
2075                 kfree(q_vector);
2076                 adapter->q_vector[q_idx] = NULL;
2077         }
2078         return -ENOMEM;
2079 }
2080
2081 /**
2082  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2083  * @adapter: board private structure to initialize
2084  *
2085  * This function frees the memory allocated to the q_vectors.  In addition if
2086  * NAPI is enabled it will delete any references to the NAPI struct prior
2087  * to freeing the q_vector.
2088  **/
2089 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2090 {
2091         int q_idx, num_q_vectors;
2092         int napi_vectors;
2093
2094         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2095         napi_vectors = adapter->num_rx_queues;
2096
2097         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2098                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2099
2100                 adapter->q_vector[q_idx] = NULL;
2101                 if (q_idx < napi_vectors)
2102                         netif_napi_del(&q_vector->napi);
2103                 kfree(q_vector);
2104         }
2105 }
2106
2107 /**
2108  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2109  * @adapter: board private structure
2110  *
2111  **/
2112 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2113 {
2114         pci_disable_msix(adapter->pdev);
2115         kfree(adapter->msix_entries);
2116         adapter->msix_entries = NULL;
2117 }
2118
2119 /**
2120  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2121  * @adapter: board private structure to initialize
2122  *
2123  **/
2124 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2125 {
2126         int err;
2127
2128         /* Number of supported queues */
2129         ixgbevf_set_num_queues(adapter);
2130
2131         err = ixgbevf_set_interrupt_capability(adapter);
2132         if (err) {
2133                 hw_dbg(&adapter->hw,
2134                        "Unable to setup interrupt capabilities\n");
2135                 goto err_set_interrupt;
2136         }
2137
2138         err = ixgbevf_alloc_q_vectors(adapter);
2139         if (err) {
2140                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2141                        "vectors\n");
2142                 goto err_alloc_q_vectors;
2143         }
2144
2145         err = ixgbevf_alloc_queues(adapter);
2146         if (err) {
2147                 pr_err("Unable to allocate memory for queues\n");
2148                 goto err_alloc_queues;
2149         }
2150
2151         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2152                "Tx Queue count = %u\n",
2153                (adapter->num_rx_queues > 1) ? "Enabled" :
2154                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2155
2156         set_bit(__IXGBEVF_DOWN, &adapter->state);
2157
2158         return 0;
2159 err_alloc_queues:
2160         ixgbevf_free_q_vectors(adapter);
2161 err_alloc_q_vectors:
2162         ixgbevf_reset_interrupt_capability(adapter);
2163 err_set_interrupt:
2164         return err;
2165 }
2166
2167 /**
2168  * ixgbevf_sw_init - Initialize general software structures
2169  * (struct ixgbevf_adapter)
2170  * @adapter: board private structure to initialize
2171  *
2172  * ixgbevf_sw_init initializes the Adapter private data structure.
2173  * Fields are initialized based on PCI device information and
2174  * OS network device settings (MTU size).
2175  **/
2176 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2177 {
2178         struct ixgbe_hw *hw = &adapter->hw;
2179         struct pci_dev *pdev = adapter->pdev;
2180         int err;
2181
2182         /* PCI config space info */
2183
2184         hw->vendor_id = pdev->vendor;
2185         hw->device_id = pdev->device;
2186         hw->revision_id = pdev->revision;
2187         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2188         hw->subsystem_device_id = pdev->subsystem_device;
2189
2190         hw->mbx.ops.init_params(hw);
2191         hw->mac.max_tx_queues = MAX_TX_QUEUES;
2192         hw->mac.max_rx_queues = MAX_RX_QUEUES;
2193         err = hw->mac.ops.reset_hw(hw);
2194         if (err) {
2195                 dev_info(&pdev->dev,
2196                          "PF still in reset state, assigning new address\n");
2197                 eth_hw_addr_random(adapter->netdev);
2198                 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
2199                         adapter->netdev->addr_len);
2200         } else {
2201                 err = hw->mac.ops.init_hw(hw);
2202                 if (err) {
2203                         pr_err("init_shared_code failed: %d\n", err);
2204                         goto out;
2205                 }
2206                 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
2207                         adapter->netdev->addr_len);
2208         }
2209
2210         /* Enable dynamic interrupt throttling rates */
2211         adapter->eitr_param = 20000;
2212         adapter->itr_setting = 1;
2213
2214         /* set defaults for eitr in MegaBytes */
2215         adapter->eitr_low = 10;
2216         adapter->eitr_high = 20;
2217
2218         /* set default ring sizes */
2219         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2220         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2221
2222         /* enable rx csum by default */
2223         adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
2224
2225         set_bit(__IXGBEVF_DOWN, &adapter->state);
2226         return 0;
2227
2228 out:
2229         return err;
2230 }
2231
2232 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
2233         {                                                       \
2234                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
2235                 if (current_counter < last_counter)             \
2236                         counter += 0x100000000LL;               \
2237                 last_counter = current_counter;                 \
2238                 counter &= 0xFFFFFFFF00000000LL;                \
2239                 counter |= current_counter;                     \
2240         }
2241
2242 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2243         {                                                                \
2244                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
2245                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
2246                 u64 current_counter = (current_counter_msb << 32) |      \
2247                         current_counter_lsb;                             \
2248                 if (current_counter < last_counter)                      \
2249                         counter += 0x1000000000LL;                       \
2250                 last_counter = current_counter;                          \
2251                 counter &= 0xFFFFFFF000000000LL;                         \
2252                 counter |= current_counter;                              \
2253         }
2254 /**
2255  * ixgbevf_update_stats - Update the board statistics counters.
2256  * @adapter: board private structure
2257  **/
2258 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2259 {
2260         struct ixgbe_hw *hw = &adapter->hw;
2261
2262         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2263                                 adapter->stats.vfgprc);
2264         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2265                                 adapter->stats.vfgptc);
2266         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2267                                 adapter->stats.last_vfgorc,
2268                                 adapter->stats.vfgorc);
2269         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2270                                 adapter->stats.last_vfgotc,
2271                                 adapter->stats.vfgotc);
2272         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2273                                 adapter->stats.vfmprc);
2274 }
2275
2276 /**
2277  * ixgbevf_watchdog - Timer Call-back
2278  * @data: pointer to adapter cast into an unsigned long
2279  **/
2280 static void ixgbevf_watchdog(unsigned long data)
2281 {
2282         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2283         struct ixgbe_hw *hw = &adapter->hw;
2284         u64 eics = 0;
2285         int i;
2286
2287         /*
2288          * Do the watchdog outside of interrupt context due to the lovely
2289          * delays that some of the newer hardware requires
2290          */
2291
2292         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2293                 goto watchdog_short_circuit;
2294
2295         /* get one bit for every active tx/rx interrupt vector */
2296         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2297                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2298                 if (qv->rxr_count || qv->txr_count)
2299                         eics |= (1 << i);
2300         }
2301
2302         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, (u32)eics);
2303
2304 watchdog_short_circuit:
2305         schedule_work(&adapter->watchdog_task);
2306 }
2307
2308 /**
2309  * ixgbevf_tx_timeout - Respond to a Tx Hang
2310  * @netdev: network interface device structure
2311  **/
2312 static void ixgbevf_tx_timeout(struct net_device *netdev)
2313 {
2314         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2315
2316         /* Do the reset outside of interrupt context */
2317         schedule_work(&adapter->reset_task);
2318 }
2319
2320 static void ixgbevf_reset_task(struct work_struct *work)
2321 {
2322         struct ixgbevf_adapter *adapter;
2323         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2324
2325         /* If we're already down or resetting, just bail */
2326         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2327             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2328                 return;
2329
2330         adapter->tx_timeout_count++;
2331
2332         ixgbevf_reinit_locked(adapter);
2333 }
2334
2335 /**
2336  * ixgbevf_watchdog_task - worker thread to bring link up
2337  * @work: pointer to work_struct containing our data
2338  **/
2339 static void ixgbevf_watchdog_task(struct work_struct *work)
2340 {
2341         struct ixgbevf_adapter *adapter = container_of(work,
2342                                                        struct ixgbevf_adapter,
2343                                                        watchdog_task);
2344         struct net_device *netdev = adapter->netdev;
2345         struct ixgbe_hw *hw = &adapter->hw;
2346         u32 link_speed = adapter->link_speed;
2347         bool link_up = adapter->link_up;
2348
2349         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2350
2351         /*
2352          * Always check the link on the watchdog because we have
2353          * no LSC interrupt
2354          */
2355         if (hw->mac.ops.check_link) {
2356                 if ((hw->mac.ops.check_link(hw, &link_speed,
2357                                             &link_up, false)) != 0) {
2358                         adapter->link_up = link_up;
2359                         adapter->link_speed = link_speed;
2360                         netif_carrier_off(netdev);
2361                         netif_tx_stop_all_queues(netdev);
2362                         schedule_work(&adapter->reset_task);
2363                         goto pf_has_reset;
2364                 }
2365         } else {
2366                 /* always assume link is up, if no check link
2367                  * function */
2368                 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2369                 link_up = true;
2370         }
2371         adapter->link_up = link_up;
2372         adapter->link_speed = link_speed;
2373
2374         if (link_up) {
2375                 if (!netif_carrier_ok(netdev)) {
2376                         hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2377                                (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2378                                10 : 1);
2379                         netif_carrier_on(netdev);
2380                         netif_tx_wake_all_queues(netdev);
2381                 }
2382         } else {
2383                 adapter->link_up = false;
2384                 adapter->link_speed = 0;
2385                 if (netif_carrier_ok(netdev)) {
2386                         hw_dbg(&adapter->hw, "NIC Link is Down\n");
2387                         netif_carrier_off(netdev);
2388                         netif_tx_stop_all_queues(netdev);
2389                 }
2390         }
2391
2392         ixgbevf_update_stats(adapter);
2393
2394 pf_has_reset:
2395         /* Reset the timer */
2396         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2397                 mod_timer(&adapter->watchdog_timer,
2398                           round_jiffies(jiffies + (2 * HZ)));
2399
2400         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2401 }
2402
2403 /**
2404  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2405  * @adapter: board private structure
2406  * @tx_ring: Tx descriptor ring for a specific queue
2407  *
2408  * Free all transmit software resources
2409  **/
2410 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2411                                struct ixgbevf_ring *tx_ring)
2412 {
2413         struct pci_dev *pdev = adapter->pdev;
2414
2415         ixgbevf_clean_tx_ring(adapter, tx_ring);
2416
2417         vfree(tx_ring->tx_buffer_info);
2418         tx_ring->tx_buffer_info = NULL;
2419
2420         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2421                           tx_ring->dma);
2422
2423         tx_ring->desc = NULL;
2424 }
2425
2426 /**
2427  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2428  * @adapter: board private structure
2429  *
2430  * Free all transmit software resources
2431  **/
2432 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2433 {
2434         int i;
2435
2436         for (i = 0; i < adapter->num_tx_queues; i++)
2437                 if (adapter->tx_ring[i].desc)
2438                         ixgbevf_free_tx_resources(adapter,
2439                                                   &adapter->tx_ring[i]);
2440
2441 }
2442
2443 /**
2444  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2445  * @adapter: board private structure
2446  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2447  *
2448  * Return 0 on success, negative on failure
2449  **/
2450 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2451                                struct ixgbevf_ring *tx_ring)
2452 {
2453         struct pci_dev *pdev = adapter->pdev;
2454         int size;
2455
2456         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2457         tx_ring->tx_buffer_info = vzalloc(size);
2458         if (!tx_ring->tx_buffer_info)
2459                 goto err;
2460
2461         /* round up to nearest 4K */
2462         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2463         tx_ring->size = ALIGN(tx_ring->size, 4096);
2464
2465         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2466                                            &tx_ring->dma, GFP_KERNEL);
2467         if (!tx_ring->desc)
2468                 goto err;
2469
2470         tx_ring->next_to_use = 0;
2471         tx_ring->next_to_clean = 0;
2472         tx_ring->work_limit = tx_ring->count;
2473         return 0;
2474
2475 err:
2476         vfree(tx_ring->tx_buffer_info);
2477         tx_ring->tx_buffer_info = NULL;
2478         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2479                "descriptor ring\n");
2480         return -ENOMEM;
2481 }
2482
2483 /**
2484  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2485  * @adapter: board private structure
2486  *
2487  * If this function returns with an error, then it's possible one or
2488  * more of the rings is populated (while the rest are not).  It is the
2489  * callers duty to clean those orphaned rings.
2490  *
2491  * Return 0 on success, negative on failure
2492  **/
2493 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2494 {
2495         int i, err = 0;
2496
2497         for (i = 0; i < adapter->num_tx_queues; i++) {
2498                 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2499                 if (!err)
2500                         continue;
2501                 hw_dbg(&adapter->hw,
2502                        "Allocation for Tx Queue %u failed\n", i);
2503                 break;
2504         }
2505
2506         return err;
2507 }
2508
2509 /**
2510  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2511  * @adapter: board private structure
2512  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2513  *
2514  * Returns 0 on success, negative on failure
2515  **/
2516 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2517                                struct ixgbevf_ring *rx_ring)
2518 {
2519         struct pci_dev *pdev = adapter->pdev;
2520         int size;
2521
2522         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2523         rx_ring->rx_buffer_info = vzalloc(size);
2524         if (!rx_ring->rx_buffer_info)
2525                 goto alloc_failed;
2526
2527         /* Round up to nearest 4K */
2528         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2529         rx_ring->size = ALIGN(rx_ring->size, 4096);
2530
2531         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2532                                            &rx_ring->dma, GFP_KERNEL);
2533
2534         if (!rx_ring->desc) {
2535                 hw_dbg(&adapter->hw,
2536                        "Unable to allocate memory for "
2537                        "the receive descriptor ring\n");
2538                 vfree(rx_ring->rx_buffer_info);
2539                 rx_ring->rx_buffer_info = NULL;
2540                 goto alloc_failed;
2541         }
2542
2543         rx_ring->next_to_clean = 0;
2544         rx_ring->next_to_use = 0;
2545
2546         return 0;
2547 alloc_failed:
2548         return -ENOMEM;
2549 }
2550
2551 /**
2552  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2553  * @adapter: board private structure
2554  *
2555  * If this function returns with an error, then it's possible one or
2556  * more of the rings is populated (while the rest are not).  It is the
2557  * callers duty to clean those orphaned rings.
2558  *
2559  * Return 0 on success, negative on failure
2560  **/
2561 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2562 {
2563         int i, err = 0;
2564
2565         for (i = 0; i < adapter->num_rx_queues; i++) {
2566                 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2567                 if (!err)
2568                         continue;
2569                 hw_dbg(&adapter->hw,
2570                        "Allocation for Rx Queue %u failed\n", i);
2571                 break;
2572         }
2573         return err;
2574 }
2575
2576 /**
2577  * ixgbevf_free_rx_resources - Free Rx Resources
2578  * @adapter: board private structure
2579  * @rx_ring: ring to clean the resources from
2580  *
2581  * Free all receive software resources
2582  **/
2583 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2584                                struct ixgbevf_ring *rx_ring)
2585 {
2586         struct pci_dev *pdev = adapter->pdev;
2587
2588         ixgbevf_clean_rx_ring(adapter, rx_ring);
2589
2590         vfree(rx_ring->rx_buffer_info);
2591         rx_ring->rx_buffer_info = NULL;
2592
2593         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2594                           rx_ring->dma);
2595
2596         rx_ring->desc = NULL;
2597 }
2598
2599 /**
2600  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2601  * @adapter: board private structure
2602  *
2603  * Free all receive software resources
2604  **/
2605 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2606 {
2607         int i;
2608
2609         for (i = 0; i < adapter->num_rx_queues; i++)
2610                 if (adapter->rx_ring[i].desc)
2611                         ixgbevf_free_rx_resources(adapter,
2612                                                   &adapter->rx_ring[i]);
2613 }
2614
2615 /**
2616  * ixgbevf_open - Called when a network interface is made active
2617  * @netdev: network interface device structure
2618  *
2619  * Returns 0 on success, negative value on failure
2620  *
2621  * The open entry point is called when a network interface is made
2622  * active by the system (IFF_UP).  At this point all resources needed
2623  * for transmit and receive operations are allocated, the interrupt
2624  * handler is registered with the OS, the watchdog timer is started,
2625  * and the stack is notified that the interface is ready.
2626  **/
2627 static int ixgbevf_open(struct net_device *netdev)
2628 {
2629         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2630         struct ixgbe_hw *hw = &adapter->hw;
2631         int err;
2632
2633         /* disallow open during test */
2634         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2635                 return -EBUSY;
2636
2637         if (hw->adapter_stopped) {
2638                 ixgbevf_reset(adapter);
2639                 /* if adapter is still stopped then PF isn't up and
2640                  * the vf can't start. */
2641                 if (hw->adapter_stopped) {
2642                         err = IXGBE_ERR_MBX;
2643                         pr_err("Unable to start - perhaps the PF Driver isn't "
2644                                "up yet\n");
2645                         goto err_setup_reset;
2646                 }
2647         }
2648
2649         /* allocate transmit descriptors */
2650         err = ixgbevf_setup_all_tx_resources(adapter);
2651         if (err)
2652                 goto err_setup_tx;
2653
2654         /* allocate receive descriptors */
2655         err = ixgbevf_setup_all_rx_resources(adapter);
2656         if (err)
2657                 goto err_setup_rx;
2658
2659         ixgbevf_configure(adapter);
2660
2661         /*
2662          * Map the Tx/Rx rings to the vectors we were allotted.
2663          * if request_irq will be called in this function map_rings
2664          * must be called *before* up_complete
2665          */
2666         ixgbevf_map_rings_to_vectors(adapter);
2667
2668         ixgbevf_up_complete(adapter);
2669
2670         /* clear any pending interrupts, may auto mask */
2671         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2672         err = ixgbevf_request_irq(adapter);
2673         if (err)
2674                 goto err_req_irq;
2675
2676         ixgbevf_irq_enable(adapter, true, true);
2677
2678         return 0;
2679
2680 err_req_irq:
2681         ixgbevf_down(adapter);
2682         ixgbevf_free_irq(adapter);
2683 err_setup_rx:
2684         ixgbevf_free_all_rx_resources(adapter);
2685 err_setup_tx:
2686         ixgbevf_free_all_tx_resources(adapter);
2687         ixgbevf_reset(adapter);
2688
2689 err_setup_reset:
2690
2691         return err;
2692 }
2693
2694 /**
2695  * ixgbevf_close - Disables a network interface
2696  * @netdev: network interface device structure
2697  *
2698  * Returns 0, this is not allowed to fail
2699  *
2700  * The close entry point is called when an interface is de-activated
2701  * by the OS.  The hardware is still under the drivers control, but
2702  * needs to be disabled.  A global MAC reset is issued to stop the
2703  * hardware, and all transmit and receive resources are freed.
2704  **/
2705 static int ixgbevf_close(struct net_device *netdev)
2706 {
2707         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2708
2709         ixgbevf_down(adapter);
2710         ixgbevf_free_irq(adapter);
2711
2712         ixgbevf_free_all_tx_resources(adapter);
2713         ixgbevf_free_all_rx_resources(adapter);
2714
2715         return 0;
2716 }
2717
2718 static int ixgbevf_tso(struct ixgbevf_adapter *adapter,
2719                        struct ixgbevf_ring *tx_ring,
2720                        struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2721 {
2722         struct ixgbe_adv_tx_context_desc *context_desc;
2723         unsigned int i;
2724         int err;
2725         struct ixgbevf_tx_buffer *tx_buffer_info;
2726         u32 vlan_macip_lens = 0, type_tucmd_mlhl;
2727         u32 mss_l4len_idx, l4len;
2728
2729         if (skb_is_gso(skb)) {
2730                 if (skb_header_cloned(skb)) {
2731                         err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2732                         if (err)
2733                                 return err;
2734                 }
2735                 l4len = tcp_hdrlen(skb);
2736                 *hdr_len += l4len;
2737
2738                 if (skb->protocol == htons(ETH_P_IP)) {
2739                         struct iphdr *iph = ip_hdr(skb);
2740                         iph->tot_len = 0;
2741                         iph->check = 0;
2742                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2743                                                                  iph->daddr, 0,
2744                                                                  IPPROTO_TCP,
2745                                                                  0);
2746                         adapter->hw_tso_ctxt++;
2747                 } else if (skb_is_gso_v6(skb)) {
2748                         ipv6_hdr(skb)->payload_len = 0;
2749                         tcp_hdr(skb)->check =
2750                             ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2751                                              &ipv6_hdr(skb)->daddr,
2752                                              0, IPPROTO_TCP, 0);
2753                         adapter->hw_tso6_ctxt++;
2754                 }
2755
2756                 i = tx_ring->next_to_use;
2757
2758                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2759                 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i);
2760
2761                 /* VLAN MACLEN IPLEN */
2762                 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2763                         vlan_macip_lens |=
2764                                 (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK);
2765                 vlan_macip_lens |= ((skb_network_offset(skb)) <<
2766                                     IXGBE_ADVTXD_MACLEN_SHIFT);
2767                 *hdr_len += skb_network_offset(skb);
2768                 vlan_macip_lens |=
2769                         (skb_transport_header(skb) - skb_network_header(skb));
2770                 *hdr_len +=
2771                         (skb_transport_header(skb) - skb_network_header(skb));
2772                 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2773                 context_desc->seqnum_seed = 0;
2774
2775                 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2776                 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT |
2777                                     IXGBE_ADVTXD_DTYP_CTXT);
2778
2779                 if (skb->protocol == htons(ETH_P_IP))
2780                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2781                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2782                 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
2783
2784                 /* MSS L4LEN IDX */
2785                 mss_l4len_idx =
2786                         (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT);
2787                 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT);
2788                 /* use index 1 for TSO */
2789                 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2790                 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2791
2792                 tx_buffer_info->time_stamp = jiffies;
2793                 tx_buffer_info->next_to_watch = i;
2794
2795                 i++;
2796                 if (i == tx_ring->count)
2797                         i = 0;
2798                 tx_ring->next_to_use = i;
2799
2800                 return true;
2801         }
2802
2803         return false;
2804 }
2805
2806 static bool ixgbevf_tx_csum(struct ixgbevf_adapter *adapter,
2807                             struct ixgbevf_ring *tx_ring,
2808                             struct sk_buff *skb, u32 tx_flags)
2809 {
2810         struct ixgbe_adv_tx_context_desc *context_desc;
2811         unsigned int i;
2812         struct ixgbevf_tx_buffer *tx_buffer_info;
2813         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2814
2815         if (skb->ip_summed == CHECKSUM_PARTIAL ||
2816             (tx_flags & IXGBE_TX_FLAGS_VLAN)) {
2817                 i = tx_ring->next_to_use;
2818                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2819                 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i);
2820
2821                 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2822                         vlan_macip_lens |= (tx_flags &
2823                                             IXGBE_TX_FLAGS_VLAN_MASK);
2824                 vlan_macip_lens |= (skb_network_offset(skb) <<
2825                                     IXGBE_ADVTXD_MACLEN_SHIFT);
2826                 if (skb->ip_summed == CHECKSUM_PARTIAL)
2827                         vlan_macip_lens |= (skb_transport_header(skb) -
2828                                             skb_network_header(skb));
2829
2830                 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2831                 context_desc->seqnum_seed = 0;
2832
2833                 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT |
2834                                     IXGBE_ADVTXD_DTYP_CTXT);
2835
2836                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2837                         switch (skb->protocol) {
2838                         case __constant_htons(ETH_P_IP):
2839                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2840                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2841                                         type_tucmd_mlhl |=
2842                                             IXGBE_ADVTXD_TUCMD_L4T_TCP;
2843                                 break;
2844                         case __constant_htons(ETH_P_IPV6):
2845                                 /* XXX what about other V6 headers?? */
2846                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2847                                         type_tucmd_mlhl |=
2848                                                 IXGBE_ADVTXD_TUCMD_L4T_TCP;
2849                                 break;
2850                         default:
2851                                 if (unlikely(net_ratelimit())) {
2852                                         pr_warn("partial checksum but "
2853                                                 "proto=%x!\n", skb->protocol);
2854                                 }
2855                                 break;
2856                         }
2857                 }
2858
2859                 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
2860                 /* use index zero for tx checksum offload */
2861                 context_desc->mss_l4len_idx = 0;
2862
2863                 tx_buffer_info->time_stamp = jiffies;
2864                 tx_buffer_info->next_to_watch = i;
2865
2866                 adapter->hw_csum_tx_good++;
2867                 i++;
2868                 if (i == tx_ring->count)
2869                         i = 0;
2870                 tx_ring->next_to_use = i;
2871
2872                 return true;
2873         }
2874
2875         return false;
2876 }
2877
2878 static int ixgbevf_tx_map(struct ixgbevf_adapter *adapter,
2879                           struct ixgbevf_ring *tx_ring,
2880                           struct sk_buff *skb, u32 tx_flags,
2881                           unsigned int first)
2882 {
2883         struct pci_dev *pdev = adapter->pdev;
2884         struct ixgbevf_tx_buffer *tx_buffer_info;
2885         unsigned int len;
2886         unsigned int total = skb->len;
2887         unsigned int offset = 0, size;
2888         int count = 0;
2889         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2890         unsigned int f;
2891         int i;
2892
2893         i = tx_ring->next_to_use;
2894
2895         len = min(skb_headlen(skb), total);
2896         while (len) {
2897                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2898                 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2899
2900                 tx_buffer_info->length = size;
2901                 tx_buffer_info->mapped_as_page = false;
2902                 tx_buffer_info->dma = dma_map_single(&adapter->pdev->dev,
2903                                                      skb->data + offset,
2904                                                      size, DMA_TO_DEVICE);
2905                 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma))
2906                         goto dma_error;
2907                 tx_buffer_info->time_stamp = jiffies;
2908                 tx_buffer_info->next_to_watch = i;
2909
2910                 len -= size;
2911                 total -= size;
2912                 offset += size;
2913                 count++;
2914                 i++;
2915                 if (i == tx_ring->count)
2916                         i = 0;
2917         }
2918
2919         for (f = 0; f < nr_frags; f++) {
2920                 const struct skb_frag_struct *frag;
2921
2922                 frag = &skb_shinfo(skb)->frags[f];
2923                 len = min((unsigned int)skb_frag_size(frag), total);
2924                 offset = 0;
2925
2926                 while (len) {
2927                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
2928                         size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2929
2930                         tx_buffer_info->length = size;
2931                         tx_buffer_info->dma =
2932                                 skb_frag_dma_map(&adapter->pdev->dev, frag,
2933                                                  offset, size, DMA_TO_DEVICE);
2934                         tx_buffer_info->mapped_as_page = true;
2935                         if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma))
2936                                 goto dma_error;
2937                         tx_buffer_info->time_stamp = jiffies;
2938                         tx_buffer_info->next_to_watch = i;
2939
2940                         len -= size;
2941                         total -= size;
2942                         offset += size;
2943                         count++;
2944                         i++;
2945                         if (i == tx_ring->count)
2946                                 i = 0;
2947                 }
2948                 if (total == 0)
2949                         break;
2950         }
2951
2952         if (i == 0)
2953                 i = tx_ring->count - 1;
2954         else
2955                 i = i - 1;
2956         tx_ring->tx_buffer_info[i].skb = skb;
2957         tx_ring->tx_buffer_info[first].next_to_watch = i;
2958
2959         return count;
2960
2961 dma_error:
2962         dev_err(&pdev->dev, "TX DMA map failed\n");
2963
2964         /* clear timestamp and dma mappings for failed tx_buffer_info map */
2965         tx_buffer_info->dma = 0;
2966         tx_buffer_info->time_stamp = 0;
2967         tx_buffer_info->next_to_watch = 0;
2968         count--;
2969
2970         /* clear timestamp and dma mappings for remaining portion of packet */
2971         while (count >= 0) {
2972                 count--;
2973                 i--;
2974                 if (i < 0)
2975                         i += tx_ring->count;
2976                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2977                 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info);
2978         }
2979
2980         return count;
2981 }
2982
2983 static void ixgbevf_tx_queue(struct ixgbevf_adapter *adapter,
2984                              struct ixgbevf_ring *tx_ring, int tx_flags,
2985                              int count, u32 paylen, u8 hdr_len)
2986 {
2987         union ixgbe_adv_tx_desc *tx_desc = NULL;
2988         struct ixgbevf_tx_buffer *tx_buffer_info;
2989         u32 olinfo_status = 0, cmd_type_len = 0;
2990         unsigned int i;
2991
2992         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2993
2994         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2995
2996         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2997
2998         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2999                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
3000
3001         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
3002                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3003
3004                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
3005                         IXGBE_ADVTXD_POPTS_SHIFT;
3006
3007                 /* use index 1 context for tso */
3008                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
3009                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3010                         olinfo_status |= IXGBE_TXD_POPTS_IXSM <<
3011                                 IXGBE_ADVTXD_POPTS_SHIFT;
3012
3013         } else if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3014                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
3015                         IXGBE_ADVTXD_POPTS_SHIFT;
3016
3017         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
3018
3019         i = tx_ring->next_to_use;
3020         while (count--) {
3021                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
3022                 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
3023                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
3024                 tx_desc->read.cmd_type_len =
3025                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
3026                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3027                 i++;
3028                 if (i == tx_ring->count)
3029                         i = 0;
3030         }
3031
3032         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
3033
3034         /*
3035          * Force memory writes to complete before letting h/w
3036          * know there are new descriptors to fetch.  (Only
3037          * applicable for weak-ordered memory model archs,
3038          * such as IA-64).
3039          */
3040         wmb();
3041
3042         tx_ring->next_to_use = i;
3043         writel(i, adapter->hw.hw_addr + tx_ring->tail);
3044 }
3045
3046 static int __ixgbevf_maybe_stop_tx(struct net_device *netdev,
3047                                    struct ixgbevf_ring *tx_ring, int size)
3048 {
3049         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3050
3051         netif_stop_subqueue(netdev, tx_ring->queue_index);
3052         /* Herbert's original patch had:
3053          *  smp_mb__after_netif_stop_queue();
3054          * but since that doesn't exist yet, just open code it. */
3055         smp_mb();
3056
3057         /* We need to check again in a case another CPU has just
3058          * made room available. */
3059         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
3060                 return -EBUSY;
3061
3062         /* A reprieve! - use start_queue because it doesn't call schedule */
3063         netif_start_subqueue(netdev, tx_ring->queue_index);
3064         ++adapter->restart_queue;
3065         return 0;
3066 }
3067
3068 static int ixgbevf_maybe_stop_tx(struct net_device *netdev,
3069                                  struct ixgbevf_ring *tx_ring, int size)
3070 {
3071         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
3072                 return 0;
3073         return __ixgbevf_maybe_stop_tx(netdev, tx_ring, size);
3074 }
3075
3076 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3077 {
3078         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3079         struct ixgbevf_ring *tx_ring;
3080         unsigned int first;
3081         unsigned int tx_flags = 0;
3082         u8 hdr_len = 0;
3083         int r_idx = 0, tso;
3084         int count = 0;
3085
3086         unsigned int f;
3087
3088         tx_ring = &adapter->tx_ring[r_idx];
3089
3090         if (vlan_tx_tag_present(skb)) {
3091                 tx_flags |= vlan_tx_tag_get(skb);
3092                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3093                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3094         }
3095
3096         /* four things can cause us to need a context descriptor */
3097         if (skb_is_gso(skb) ||
3098             (skb->ip_summed == CHECKSUM_PARTIAL) ||
3099             (tx_flags & IXGBE_TX_FLAGS_VLAN))
3100                 count++;
3101
3102         count += TXD_USE_COUNT(skb_headlen(skb));
3103         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3104                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]));
3105
3106         if (ixgbevf_maybe_stop_tx(netdev, tx_ring, count)) {
3107                 adapter->tx_busy++;
3108                 return NETDEV_TX_BUSY;
3109         }
3110
3111         first = tx_ring->next_to_use;
3112
3113         if (skb->protocol == htons(ETH_P_IP))
3114                 tx_flags |= IXGBE_TX_FLAGS_IPV4;
3115         tso = ixgbevf_tso(adapter, tx_ring, skb, tx_flags, &hdr_len);
3116         if (tso < 0) {
3117                 dev_kfree_skb_any(skb);
3118                 return NETDEV_TX_OK;
3119         }
3120
3121         if (tso)
3122                 tx_flags |= IXGBE_TX_FLAGS_TSO;
3123         else if (ixgbevf_tx_csum(adapter, tx_ring, skb, tx_flags) &&
3124                  (skb->ip_summed == CHECKSUM_PARTIAL))
3125                 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3126
3127         ixgbevf_tx_queue(adapter, tx_ring, tx_flags,
3128                          ixgbevf_tx_map(adapter, tx_ring, skb, tx_flags, first),
3129                          skb->len, hdr_len);
3130
3131         ixgbevf_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED);
3132
3133         return NETDEV_TX_OK;
3134 }
3135
3136 /**
3137  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3138  * @netdev: network interface device structure
3139  * @p: pointer to an address structure
3140  *
3141  * Returns 0 on success, negative on failure
3142  **/
3143 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3144 {
3145         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3146         struct ixgbe_hw *hw = &adapter->hw;
3147         struct sockaddr *addr = p;
3148
3149         if (!is_valid_ether_addr(addr->sa_data))
3150                 return -EADDRNOTAVAIL;
3151
3152         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3153         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3154
3155         if (hw->mac.ops.set_rar)
3156                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3157
3158         return 0;
3159 }
3160
3161 /**
3162  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3163  * @netdev: network interface device structure
3164  * @new_mtu: new value for maximum frame size
3165  *
3166  * Returns 0 on success, negative on failure
3167  **/
3168 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3169 {
3170         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3171         struct ixgbe_hw *hw = &adapter->hw;
3172         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3173         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3174         u32 msg[2];
3175
3176         if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3177                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3178
3179         /* MTU < 68 is an error and causes problems on some kernels */
3180         if ((new_mtu < 68) || (max_frame > max_possible_frame))
3181                 return -EINVAL;
3182
3183         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3184                netdev->mtu, new_mtu);
3185         /* must set new MTU before calling down or up */
3186         netdev->mtu = new_mtu;
3187
3188         if (!netif_running(netdev)) {
3189                 msg[0] = IXGBE_VF_SET_LPE;
3190                 msg[1] = max_frame;
3191                 hw->mbx.ops.write_posted(hw, msg, 2);
3192         }
3193
3194         if (netif_running(netdev))
3195                 ixgbevf_reinit_locked(adapter);
3196
3197         return 0;
3198 }
3199
3200 static void ixgbevf_shutdown(struct pci_dev *pdev)
3201 {
3202         struct net_device *netdev = pci_get_drvdata(pdev);
3203         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3204
3205         netif_device_detach(netdev);
3206
3207         if (netif_running(netdev)) {
3208                 ixgbevf_down(adapter);
3209                 ixgbevf_free_irq(adapter);
3210                 ixgbevf_free_all_tx_resources(adapter);
3211                 ixgbevf_free_all_rx_resources(adapter);
3212         }
3213
3214 #ifdef CONFIG_PM
3215         pci_save_state(pdev);
3216 #endif
3217
3218         pci_disable_device(pdev);
3219 }
3220
3221 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3222                                                 struct rtnl_link_stats64 *stats)
3223 {
3224         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3225         unsigned int start;
3226         u64 bytes, packets;
3227         const struct ixgbevf_ring *ring;
3228         int i;
3229
3230         ixgbevf_update_stats(adapter);
3231
3232         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3233
3234         for (i = 0; i < adapter->num_rx_queues; i++) {
3235                 ring = &adapter->rx_ring[i];
3236                 do {
3237                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3238                         bytes = ring->total_bytes;
3239                         packets = ring->total_packets;
3240                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3241                 stats->rx_bytes += bytes;
3242                 stats->rx_packets += packets;
3243         }
3244
3245         for (i = 0; i < adapter->num_tx_queues; i++) {
3246                 ring = &adapter->tx_ring[i];
3247                 do {
3248                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3249                         bytes = ring->total_bytes;
3250                         packets = ring->total_packets;
3251                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3252                 stats->tx_bytes += bytes;
3253                 stats->tx_packets += packets;
3254         }
3255
3256         return stats;
3257 }
3258
3259 static int ixgbevf_set_features(struct net_device *netdev,
3260         netdev_features_t features)
3261 {
3262         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3263
3264         if (features & NETIF_F_RXCSUM)
3265                 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
3266         else
3267                 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
3268
3269         return 0;
3270 }
3271
3272 static const struct net_device_ops ixgbe_netdev_ops = {
3273         .ndo_open               = ixgbevf_open,
3274         .ndo_stop               = ixgbevf_close,
3275         .ndo_start_xmit         = ixgbevf_xmit_frame,
3276         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3277         .ndo_get_stats64        = ixgbevf_get_stats,
3278         .ndo_validate_addr      = eth_validate_addr,
3279         .ndo_set_mac_address    = ixgbevf_set_mac,
3280         .ndo_change_mtu         = ixgbevf_change_mtu,
3281         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3282         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3283         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3284         .ndo_set_features       = ixgbevf_set_features,
3285 };
3286
3287 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3288 {
3289         dev->netdev_ops = &ixgbe_netdev_ops;
3290         ixgbevf_set_ethtool_ops(dev);
3291         dev->watchdog_timeo = 5 * HZ;
3292 }
3293
3294 /**
3295  * ixgbevf_probe - Device Initialization Routine
3296  * @pdev: PCI device information struct
3297  * @ent: entry in ixgbevf_pci_tbl
3298  *
3299  * Returns 0 on success, negative on failure
3300  *
3301  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3302  * The OS initialization, configuring of the adapter private structure,
3303  * and a hardware reset occur.
3304  **/
3305 static int __devinit ixgbevf_probe(struct pci_dev *pdev,
3306                                    const struct pci_device_id *ent)
3307 {
3308         struct net_device *netdev;
3309         struct ixgbevf_adapter *adapter = NULL;
3310         struct ixgbe_hw *hw = NULL;
3311         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3312         static int cards_found;
3313         int err, pci_using_dac;
3314
3315         err = pci_enable_device(pdev);
3316         if (err)
3317                 return err;
3318
3319         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3320             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
3321                 pci_using_dac = 1;
3322         } else {
3323                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
3324                 if (err) {
3325                         err = dma_set_coherent_mask(&pdev->dev,
3326                                                     DMA_BIT_MASK(32));
3327                         if (err) {
3328                                 dev_err(&pdev->dev, "No usable DMA "
3329                                         "configuration, aborting\n");
3330                                 goto err_dma;
3331                         }
3332                 }
3333                 pci_using_dac = 0;
3334         }
3335
3336         err = pci_request_regions(pdev, ixgbevf_driver_name);
3337         if (err) {
3338                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3339                 goto err_pci_reg;
3340         }
3341
3342         pci_set_master(pdev);
3343
3344 #ifdef HAVE_TX_MQ
3345         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3346                                    MAX_TX_QUEUES);
3347 #else
3348         netdev = alloc_etherdev(sizeof(struct ixgbevf_adapter));
3349 #endif
3350         if (!netdev) {
3351                 err = -ENOMEM;
3352                 goto err_alloc_etherdev;
3353         }
3354
3355         SET_NETDEV_DEV(netdev, &pdev->dev);
3356
3357         pci_set_drvdata(pdev, netdev);
3358         adapter = netdev_priv(netdev);
3359
3360         adapter->netdev = netdev;
3361         adapter->pdev = pdev;
3362         hw = &adapter->hw;
3363         hw->back = adapter;
3364         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3365
3366         /*
3367          * call save state here in standalone driver because it relies on
3368          * adapter struct to exist, and needs to call netdev_priv
3369          */
3370         pci_save_state(pdev);
3371
3372         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3373                               pci_resource_len(pdev, 0));
3374         if (!hw->hw_addr) {
3375                 err = -EIO;
3376                 goto err_ioremap;
3377         }
3378
3379         ixgbevf_assign_netdev_ops(netdev);
3380
3381         adapter->bd_number = cards_found;
3382
3383         /* Setup hw api */
3384         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3385         hw->mac.type  = ii->mac;
3386
3387         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3388                sizeof(struct ixgbe_mbx_operations));
3389
3390         adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE;
3391         adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3392         adapter->flags |= IXGBE_FLAG_RX_1BUF_CAPABLE;
3393
3394         /* setup the private structure */
3395         err = ixgbevf_sw_init(adapter);
3396         if (err)
3397                 goto err_sw_init;
3398
3399         /* The HW MAC address was set and/or determined in sw_init */
3400         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3401
3402         if (!is_valid_ether_addr(netdev->dev_addr)) {
3403                 pr_err("invalid MAC address\n");
3404                 err = -EIO;
3405                 goto err_sw_init;
3406         }
3407
3408         netdev->hw_features = NETIF_F_SG |
3409                            NETIF_F_IP_CSUM |
3410                            NETIF_F_IPV6_CSUM |
3411                            NETIF_F_TSO |
3412                            NETIF_F_TSO6 |
3413                            NETIF_F_RXCSUM;
3414
3415         netdev->features = netdev->hw_features |
3416                            NETIF_F_HW_VLAN_TX |
3417                            NETIF_F_HW_VLAN_RX |
3418                            NETIF_F_HW_VLAN_FILTER;
3419
3420         netdev->vlan_features |= NETIF_F_TSO;
3421         netdev->vlan_features |= NETIF_F_TSO6;
3422         netdev->vlan_features |= NETIF_F_IP_CSUM;
3423         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3424         netdev->vlan_features |= NETIF_F_SG;
3425
3426         if (pci_using_dac)
3427                 netdev->features |= NETIF_F_HIGHDMA;
3428
3429         netdev->priv_flags |= IFF_UNICAST_FLT;
3430
3431         init_timer(&adapter->watchdog_timer);
3432         adapter->watchdog_timer.function = ixgbevf_watchdog;
3433         adapter->watchdog_timer.data = (unsigned long)adapter;
3434
3435         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3436         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3437
3438         err = ixgbevf_init_interrupt_scheme(adapter);
3439         if (err)
3440                 goto err_sw_init;
3441
3442         /* pick up the PCI bus settings for reporting later */
3443         if (hw->mac.ops.get_bus_info)
3444                 hw->mac.ops.get_bus_info(hw);
3445
3446         strcpy(netdev->name, "eth%d");
3447
3448         err = register_netdev(netdev);
3449         if (err)
3450                 goto err_register;
3451
3452         adapter->netdev_registered = true;
3453
3454         netif_carrier_off(netdev);
3455
3456         ixgbevf_init_last_counter_stats(adapter);
3457
3458         /* print the MAC address */
3459         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3460
3461         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3462
3463         hw_dbg(hw, "LRO is disabled\n");
3464
3465         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3466         cards_found++;
3467         return 0;
3468
3469 err_register:
3470 err_sw_init:
3471         ixgbevf_reset_interrupt_capability(adapter);
3472         iounmap(hw->hw_addr);
3473 err_ioremap:
3474         free_netdev(netdev);
3475 err_alloc_etherdev:
3476         pci_release_regions(pdev);
3477 err_pci_reg:
3478 err_dma:
3479         pci_disable_device(pdev);
3480         return err;
3481 }
3482
3483 /**
3484  * ixgbevf_remove - Device Removal Routine
3485  * @pdev: PCI device information struct
3486  *
3487  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3488  * that it should release a PCI device.  The could be caused by a
3489  * Hot-Plug event, or because the driver is going to be removed from
3490  * memory.
3491  **/
3492 static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3493 {
3494         struct net_device *netdev = pci_get_drvdata(pdev);
3495         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3496
3497         set_bit(__IXGBEVF_DOWN, &adapter->state);
3498
3499         del_timer_sync(&adapter->watchdog_timer);
3500
3501         cancel_work_sync(&adapter->reset_task);
3502         cancel_work_sync(&adapter->watchdog_task);
3503
3504         if (adapter->netdev_registered) {
3505                 unregister_netdev(netdev);
3506                 adapter->netdev_registered = false;
3507         }
3508
3509         ixgbevf_reset_interrupt_capability(adapter);
3510
3511         iounmap(adapter->hw.hw_addr);
3512         pci_release_regions(pdev);
3513
3514         hw_dbg(&adapter->hw, "Remove complete\n");
3515
3516         kfree(adapter->tx_ring);
3517         kfree(adapter->rx_ring);
3518
3519         free_netdev(netdev);
3520
3521         pci_disable_device(pdev);
3522 }
3523
3524 static struct pci_driver ixgbevf_driver = {
3525         .name     = ixgbevf_driver_name,
3526         .id_table = ixgbevf_pci_tbl,
3527         .probe    = ixgbevf_probe,
3528         .remove   = __devexit_p(ixgbevf_remove),
3529         .shutdown = ixgbevf_shutdown,
3530 };
3531
3532 /**
3533  * ixgbevf_init_module - Driver Registration Routine
3534  *
3535  * ixgbevf_init_module is the first routine called when the driver is
3536  * loaded. All it does is register with the PCI subsystem.
3537  **/
3538 static int __init ixgbevf_init_module(void)
3539 {
3540         int ret;
3541         pr_info("%s - version %s\n", ixgbevf_driver_string,
3542                 ixgbevf_driver_version);
3543
3544         pr_info("%s\n", ixgbevf_copyright);
3545
3546         ret = pci_register_driver(&ixgbevf_driver);
3547         return ret;
3548 }
3549
3550 module_init(ixgbevf_init_module);
3551
3552 /**
3553  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3554  *
3555  * ixgbevf_exit_module is called just before the driver is removed
3556  * from memory.
3557  **/
3558 static void __exit ixgbevf_exit_module(void)
3559 {
3560         pci_unregister_driver(&ixgbevf_driver);
3561 }
3562
3563 #ifdef DEBUG
3564 /**
3565  * ixgbevf_get_hw_dev_name - return device name string
3566  * used by hardware layer to print debugging information
3567  **/
3568 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3569 {
3570         struct ixgbevf_adapter *adapter = hw->back;
3571         return adapter->netdev->name;
3572 }
3573
3574 #endif
3575 module_exit(ixgbevf_exit_module);
3576
3577 /* ixgbevf_main.c */