1 /*******************************************************************************
3 Intel 82599 Virtual Function driver
4 Copyright(c) 1999 - 2012 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
29 /******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31 ******************************************************************************/
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
35 #include <linux/types.h>
36 #include <linux/bitops.h>
37 #include <linux/module.h>
38 #include <linux/pci.h>
39 #include <linux/netdevice.h>
40 #include <linux/vmalloc.h>
41 #include <linux/string.h>
44 #include <linux/tcp.h>
45 #include <linux/sctp.h>
46 #include <linux/ipv6.h>
47 #include <linux/slab.h>
48 #include <net/checksum.h>
49 #include <net/ip6_checksum.h>
50 #include <linux/ethtool.h>
52 #include <linux/if_vlan.h>
53 #include <linux/prefetch.h>
57 const char ixgbevf_driver_name[] = "ixgbevf";
58 static const char ixgbevf_driver_string[] =
59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
61 #define DRV_VERSION "2.6.0-k"
62 const char ixgbevf_driver_version[] = DRV_VERSION;
63 static char ixgbevf_copyright[] =
64 "Copyright (c) 2009 - 2012 Intel Corporation.";
66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
71 /* ixgbevf_pci_tbl - PCI Device ID Table
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
79 static struct pci_device_id ixgbevf_pci_tbl[] = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
85 /* required last entry */
88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
92 MODULE_LICENSE("GPL");
93 MODULE_VERSION(DRV_VERSION);
95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
96 static int debug = -1;
97 module_param(debug, int, 0);
98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
103 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
104 struct ixgbevf_ring *rx_ring,
108 * Force memory writes to complete before letting h/w
109 * know there are new descriptors to fetch. (Only
110 * applicable for weak-ordered memory model archs,
114 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
118 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
119 * @adapter: pointer to adapter struct
120 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
121 * @queue: queue to map the corresponding interrupt to
122 * @msix_vector: the vector to map to the corresponding queue
125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
126 u8 queue, u8 msix_vector)
129 struct ixgbe_hw *hw = &adapter->hw;
130 if (direction == -1) {
132 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
133 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
136 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
138 /* tx or rx causes */
139 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
140 index = ((16 * (queue & 1)) + (8 * direction));
141 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
142 ivar &= ~(0xFF << index);
143 ivar |= (msix_vector << index);
144 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
149 struct ixgbevf_tx_buffer
152 if (tx_buffer_info->dma) {
153 if (tx_buffer_info->mapped_as_page)
154 dma_unmap_page(tx_ring->dev,
156 tx_buffer_info->length,
159 dma_unmap_single(tx_ring->dev,
161 tx_buffer_info->length,
163 tx_buffer_info->dma = 0;
165 if (tx_buffer_info->skb) {
166 dev_kfree_skb_any(tx_buffer_info->skb);
167 tx_buffer_info->skb = NULL;
169 tx_buffer_info->time_stamp = 0;
170 /* tx_buffer_info must be completely set up in the transmit path */
173 #define IXGBE_MAX_TXD_PWR 14
174 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
176 /* Tx Descriptors needed, worst case */
177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
180 static void ixgbevf_tx_timeout(struct net_device *netdev);
183 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
184 * @q_vector: board private structure
185 * @tx_ring: tx ring to clean
187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
188 struct ixgbevf_ring *tx_ring)
190 struct ixgbevf_adapter *adapter = q_vector->adapter;
191 union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
192 struct ixgbevf_tx_buffer *tx_buffer_info;
193 unsigned int i, eop, count = 0;
194 unsigned int total_bytes = 0, total_packets = 0;
196 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
199 i = tx_ring->next_to_clean;
200 eop = tx_ring->tx_buffer_info[i].next_to_watch;
201 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
203 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
204 (count < tx_ring->count)) {
205 bool cleaned = false;
206 rmb(); /* read buffer_info after eop_desc */
207 /* eop could change between read and DD-check */
208 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
210 for ( ; !cleaned; count++) {
212 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
213 tx_buffer_info = &tx_ring->tx_buffer_info[i];
214 cleaned = (i == eop);
215 skb = tx_buffer_info->skb;
217 if (cleaned && skb) {
218 unsigned int segs, bytecount;
220 /* gso_segs is currently only valid for tcp */
221 segs = skb_shinfo(skb)->gso_segs ?: 1;
222 /* multiply data chunks by size of headers */
223 bytecount = ((segs - 1) * skb_headlen(skb)) +
225 total_packets += segs;
226 total_bytes += bytecount;
229 ixgbevf_unmap_and_free_tx_resource(tx_ring,
232 tx_desc->wb.status = 0;
235 if (i == tx_ring->count)
240 eop = tx_ring->tx_buffer_info[i].next_to_watch;
241 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
244 tx_ring->next_to_clean = i;
246 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
247 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
248 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
249 /* Make sure that anybody stopping the queue after this
250 * sees the new next_to_clean.
253 if (__netif_subqueue_stopped(tx_ring->netdev,
254 tx_ring->queue_index) &&
255 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
256 netif_wake_subqueue(tx_ring->netdev,
257 tx_ring->queue_index);
258 ++adapter->restart_queue;
262 u64_stats_update_begin(&tx_ring->syncp);
263 tx_ring->total_bytes += total_bytes;
264 tx_ring->total_packets += total_packets;
265 u64_stats_update_end(&tx_ring->syncp);
266 q_vector->tx.total_bytes += total_bytes;
267 q_vector->tx.total_packets += total_packets;
269 return count < tx_ring->count;
273 * ixgbevf_receive_skb - Send a completed packet up the stack
274 * @q_vector: structure containing interrupt and ring information
275 * @skb: packet to send up
276 * @status: hardware indication of status of receive
277 * @rx_desc: rx descriptor
279 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
280 struct sk_buff *skb, u8 status,
281 union ixgbe_adv_rx_desc *rx_desc)
283 struct ixgbevf_adapter *adapter = q_vector->adapter;
284 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
285 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
287 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
288 __vlan_hwaccel_put_tag(skb, tag);
290 napi_gro_receive(&q_vector->napi, skb);
294 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
295 * @adapter: address of board private structure
296 * @status_err: hardware indication of status of receive
297 * @skb: skb currently being received and modified
299 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
300 struct ixgbevf_ring *ring,
301 u32 status_err, struct sk_buff *skb)
303 skb_checksum_none_assert(skb);
305 /* Rx csum disabled */
306 if (!(ring->netdev->features & NETIF_F_RXCSUM))
309 /* if IP and error */
310 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
311 (status_err & IXGBE_RXDADV_ERR_IPE)) {
312 adapter->hw_csum_rx_error++;
316 if (!(status_err & IXGBE_RXD_STAT_L4CS))
319 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
320 adapter->hw_csum_rx_error++;
324 /* It must be a TCP or UDP packet with a valid checksum */
325 skb->ip_summed = CHECKSUM_UNNECESSARY;
326 adapter->hw_csum_rx_good++;
330 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
331 * @adapter: address of board private structure
333 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
334 struct ixgbevf_ring *rx_ring,
337 struct pci_dev *pdev = adapter->pdev;
338 union ixgbe_adv_rx_desc *rx_desc;
339 struct ixgbevf_rx_buffer *bi;
341 unsigned int i = rx_ring->next_to_use;
343 bi = &rx_ring->rx_buffer_info[i];
345 while (cleaned_count--) {
346 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
349 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
350 rx_ring->rx_buf_len);
352 adapter->alloc_rx_buff_failed++;
358 bi->dma = dma_map_single(&pdev->dev, skb->data,
362 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
365 if (i == rx_ring->count)
367 bi = &rx_ring->rx_buffer_info[i];
371 if (rx_ring->next_to_use != i) {
372 rx_ring->next_to_use = i;
374 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
378 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
381 struct ixgbe_hw *hw = &adapter->hw;
383 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
386 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
387 struct ixgbevf_ring *rx_ring,
390 struct ixgbevf_adapter *adapter = q_vector->adapter;
391 struct pci_dev *pdev = adapter->pdev;
392 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
393 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
397 int cleaned_count = 0;
398 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
400 i = rx_ring->next_to_clean;
401 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
402 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
403 rx_buffer_info = &rx_ring->rx_buffer_info[i];
405 while (staterr & IXGBE_RXD_STAT_DD) {
410 rmb(); /* read descriptor and rx_buffer_info after status DD */
411 len = le16_to_cpu(rx_desc->wb.upper.length);
412 skb = rx_buffer_info->skb;
413 prefetch(skb->data - NET_IP_ALIGN);
414 rx_buffer_info->skb = NULL;
416 if (rx_buffer_info->dma) {
417 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
420 rx_buffer_info->dma = 0;
425 if (i == rx_ring->count)
428 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
432 next_buffer = &rx_ring->rx_buffer_info[i];
434 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
435 skb->next = next_buffer->skb;
436 IXGBE_CB(skb->next)->prev = skb;
437 adapter->non_eop_descs++;
441 /* we should not be chaining buffers, if we did drop the skb */
442 if (IXGBE_CB(skb)->prev) {
444 struct sk_buff *this = skb;
445 skb = IXGBE_CB(skb)->prev;
451 /* ERR_MASK will only have valid bits if EOP set */
452 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
453 dev_kfree_skb_irq(skb);
457 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb);
459 /* probably a little skewed due to removing CRC */
460 total_rx_bytes += skb->len;
464 * Work around issue of some types of VM to VM loop back
465 * packets not getting split correctly
467 if (staterr & IXGBE_RXD_STAT_LB) {
468 u32 header_fixup_len = skb_headlen(skb);
469 if (header_fixup_len < 14)
470 skb_push(skb, header_fixup_len);
472 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
474 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc);
477 rx_desc->wb.upper.status_error = 0;
479 /* return some buffers to hardware, one at a time is too slow */
480 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
481 ixgbevf_alloc_rx_buffers(adapter, rx_ring,
486 /* use prefetched values */
488 rx_buffer_info = &rx_ring->rx_buffer_info[i];
490 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
493 rx_ring->next_to_clean = i;
494 cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
497 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
499 u64_stats_update_begin(&rx_ring->syncp);
500 rx_ring->total_packets += total_rx_packets;
501 rx_ring->total_bytes += total_rx_bytes;
502 u64_stats_update_end(&rx_ring->syncp);
503 q_vector->rx.total_packets += total_rx_packets;
504 q_vector->rx.total_bytes += total_rx_bytes;
510 * ixgbevf_poll - NAPI polling calback
511 * @napi: napi struct with our devices info in it
512 * @budget: amount of work driver is allowed to do this pass, in packets
514 * This function will clean more than one or more rings associated with a
517 static int ixgbevf_poll(struct napi_struct *napi, int budget)
519 struct ixgbevf_q_vector *q_vector =
520 container_of(napi, struct ixgbevf_q_vector, napi);
521 struct ixgbevf_adapter *adapter = q_vector->adapter;
522 struct ixgbevf_ring *ring;
524 bool clean_complete = true;
526 ixgbevf_for_each_ring(ring, q_vector->tx)
527 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
529 /* attempt to distribute budget to each queue fairly, but don't allow
530 * the budget to go below 1 because we'll exit polling */
531 if (q_vector->rx.count > 1)
532 per_ring_budget = max(budget/q_vector->rx.count, 1);
534 per_ring_budget = budget;
536 ixgbevf_for_each_ring(ring, q_vector->rx)
537 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
540 /* If all work not completed, return budget and keep polling */
543 /* all work done, exit the polling mode */
545 if (adapter->rx_itr_setting & 1)
546 ixgbevf_set_itr(q_vector);
547 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
548 ixgbevf_irq_enable_queues(adapter,
549 1 << q_vector->v_idx);
555 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
556 * @q_vector: structure containing interrupt and ring information
558 static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
560 struct ixgbevf_adapter *adapter = q_vector->adapter;
561 struct ixgbe_hw *hw = &adapter->hw;
562 int v_idx = q_vector->v_idx;
563 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
566 * set the WDIS bit to not clear the timer bits and cause an
567 * immediate assertion of the interrupt
569 itr_reg |= IXGBE_EITR_CNT_WDIS;
571 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
575 * ixgbevf_configure_msix - Configure MSI-X hardware
576 * @adapter: board private structure
578 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
581 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
583 struct ixgbevf_q_vector *q_vector;
584 int q_vectors, v_idx;
586 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
587 adapter->eims_enable_mask = 0;
590 * Populate the IVAR table and set the ITR values to the
591 * corresponding register.
593 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
594 struct ixgbevf_ring *ring;
595 q_vector = adapter->q_vector[v_idx];
597 ixgbevf_for_each_ring(ring, q_vector->rx)
598 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
600 ixgbevf_for_each_ring(ring, q_vector->tx)
601 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
603 if (q_vector->tx.ring && !q_vector->rx.ring) {
605 if (adapter->tx_itr_setting == 1)
606 q_vector->itr = IXGBE_10K_ITR;
608 q_vector->itr = adapter->tx_itr_setting;
610 /* rx or rx/tx vector */
611 if (adapter->rx_itr_setting == 1)
612 q_vector->itr = IXGBE_20K_ITR;
614 q_vector->itr = adapter->rx_itr_setting;
617 /* add q_vector eims value to global eims_enable_mask */
618 adapter->eims_enable_mask |= 1 << v_idx;
620 ixgbevf_write_eitr(q_vector);
623 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
624 /* setup eims_other and add value to global eims_enable_mask */
625 adapter->eims_other = 1 << v_idx;
626 adapter->eims_enable_mask |= adapter->eims_other;
633 latency_invalid = 255
637 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
638 * @q_vector: structure containing interrupt and ring information
639 * @ring_container: structure containing ring performance data
641 * Stores a new ITR value based on packets and byte
642 * counts during the last interrupt. The advantage of per interrupt
643 * computation is faster updates and more accurate ITR for the current
644 * traffic pattern. Constants in this function were computed
645 * based on theoretical maximum wire speed and thresholds were set based
646 * on testing data as well as attempting to minimize response time
647 * while increasing bulk throughput.
649 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
650 struct ixgbevf_ring_container *ring_container)
652 int bytes = ring_container->total_bytes;
653 int packets = ring_container->total_packets;
656 u8 itr_setting = ring_container->itr;
661 /* simple throttlerate management
662 * 0-20MB/s lowest (100000 ints/s)
663 * 20-100MB/s low (20000 ints/s)
664 * 100-1249MB/s bulk (8000 ints/s)
666 /* what was last interrupt timeslice? */
667 timepassed_us = q_vector->itr >> 2;
668 bytes_perint = bytes / timepassed_us; /* bytes/usec */
670 switch (itr_setting) {
672 if (bytes_perint > 10)
673 itr_setting = low_latency;
676 if (bytes_perint > 20)
677 itr_setting = bulk_latency;
678 else if (bytes_perint <= 10)
679 itr_setting = lowest_latency;
682 if (bytes_perint <= 20)
683 itr_setting = low_latency;
687 /* clear work counters since we have the values we need */
688 ring_container->total_bytes = 0;
689 ring_container->total_packets = 0;
691 /* write updated itr to ring container */
692 ring_container->itr = itr_setting;
695 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
697 u32 new_itr = q_vector->itr;
700 ixgbevf_update_itr(q_vector, &q_vector->tx);
701 ixgbevf_update_itr(q_vector, &q_vector->rx);
703 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
705 switch (current_itr) {
706 /* counts and packets in update_itr are dependent on these numbers */
708 new_itr = IXGBE_100K_ITR;
711 new_itr = IXGBE_20K_ITR;
715 new_itr = IXGBE_8K_ITR;
719 if (new_itr != q_vector->itr) {
720 /* do an exponential smoothing */
721 new_itr = (10 * new_itr * q_vector->itr) /
722 ((9 * new_itr) + q_vector->itr);
724 /* save the algorithm value here */
725 q_vector->itr = new_itr;
727 ixgbevf_write_eitr(q_vector);
731 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
733 struct ixgbevf_adapter *adapter = data;
734 struct ixgbe_hw *hw = &adapter->hw;
736 hw->mac.get_link_status = 1;
738 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
739 mod_timer(&adapter->watchdog_timer, jiffies);
741 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
748 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
750 * @data: pointer to our q_vector struct for this interrupt vector
752 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
754 struct ixgbevf_q_vector *q_vector = data;
756 /* EIAM disabled interrupts (on this vector) for us */
757 if (q_vector->rx.ring || q_vector->tx.ring)
758 napi_schedule(&q_vector->napi);
763 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
766 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
768 a->rx_ring[r_idx].next = q_vector->rx.ring;
769 q_vector->rx.ring = &a->rx_ring[r_idx];
770 q_vector->rx.count++;
773 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
776 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
778 a->tx_ring[t_idx].next = q_vector->tx.ring;
779 q_vector->tx.ring = &a->tx_ring[t_idx];
780 q_vector->tx.count++;
784 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
785 * @adapter: board private structure to initialize
787 * This function maps descriptor rings to the queue-specific vectors
788 * we were allotted through the MSI-X enabling code. Ideally, we'd have
789 * one vector per ring/queue, but on a constrained vector budget, we
790 * group the rings as "efficiently" as possible. You would add new
791 * mapping configurations in here.
793 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
797 int rxr_idx = 0, txr_idx = 0;
798 int rxr_remaining = adapter->num_rx_queues;
799 int txr_remaining = adapter->num_tx_queues;
804 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
807 * The ideal configuration...
808 * We have enough vectors to map one per queue.
810 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
811 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
812 map_vector_to_rxq(adapter, v_start, rxr_idx);
814 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
815 map_vector_to_txq(adapter, v_start, txr_idx);
820 * If we don't have enough vectors for a 1-to-1
821 * mapping, we'll have to group them so there are
822 * multiple queues per vector.
824 /* Re-adjusting *qpv takes care of the remainder. */
825 for (i = v_start; i < q_vectors; i++) {
826 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
827 for (j = 0; j < rqpv; j++) {
828 map_vector_to_rxq(adapter, i, rxr_idx);
833 for (i = v_start; i < q_vectors; i++) {
834 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
835 for (j = 0; j < tqpv; j++) {
836 map_vector_to_txq(adapter, i, txr_idx);
847 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
848 * @adapter: board private structure
850 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
851 * interrupts from the kernel.
853 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
855 struct net_device *netdev = adapter->netdev;
856 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
860 for (vector = 0; vector < q_vectors; vector++) {
861 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
862 struct msix_entry *entry = &adapter->msix_entries[vector];
864 if (q_vector->tx.ring && q_vector->rx.ring) {
865 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
866 "%s-%s-%d", netdev->name, "TxRx", ri++);
868 } else if (q_vector->rx.ring) {
869 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
870 "%s-%s-%d", netdev->name, "rx", ri++);
871 } else if (q_vector->tx.ring) {
872 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
873 "%s-%s-%d", netdev->name, "tx", ti++);
875 /* skip this unused q_vector */
878 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
879 q_vector->name, q_vector);
882 "request_irq failed for MSIX interrupt "
884 goto free_queue_irqs;
888 err = request_irq(adapter->msix_entries[vector].vector,
889 &ixgbevf_msix_other, 0, netdev->name, adapter);
892 "request_irq for msix_other failed: %d\n", err);
893 goto free_queue_irqs;
901 free_irq(adapter->msix_entries[vector].vector,
902 adapter->q_vector[vector]);
904 pci_disable_msix(adapter->pdev);
905 kfree(adapter->msix_entries);
906 adapter->msix_entries = NULL;
910 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
912 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
914 for (i = 0; i < q_vectors; i++) {
915 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
916 q_vector->rx.ring = NULL;
917 q_vector->tx.ring = NULL;
918 q_vector->rx.count = 0;
919 q_vector->tx.count = 0;
924 * ixgbevf_request_irq - initialize interrupts
925 * @adapter: board private structure
927 * Attempts to configure interrupts using the best available
928 * capabilities of the hardware and kernel.
930 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
934 err = ixgbevf_request_msix_irqs(adapter);
938 "request_irq failed, Error %d\n", err);
943 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
947 q_vectors = adapter->num_msix_vectors;
950 free_irq(adapter->msix_entries[i].vector, adapter);
953 for (; i >= 0; i--) {
954 /* free only the irqs that were actually requested */
955 if (!adapter->q_vector[i]->rx.ring &&
956 !adapter->q_vector[i]->tx.ring)
959 free_irq(adapter->msix_entries[i].vector,
960 adapter->q_vector[i]);
963 ixgbevf_reset_q_vectors(adapter);
967 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
968 * @adapter: board private structure
970 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
972 struct ixgbe_hw *hw = &adapter->hw;
975 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
976 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
977 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
979 IXGBE_WRITE_FLUSH(hw);
981 for (i = 0; i < adapter->num_msix_vectors; i++)
982 synchronize_irq(adapter->msix_entries[i].vector);
986 * ixgbevf_irq_enable - Enable default interrupt generation settings
987 * @adapter: board private structure
989 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
991 struct ixgbe_hw *hw = &adapter->hw;
993 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
994 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
995 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
999 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1000 * @adapter: board private structure
1002 * Configure the Tx unit of the MAC after a reset.
1004 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1007 struct ixgbe_hw *hw = &adapter->hw;
1008 u32 i, j, tdlen, txctrl;
1010 /* Setup the HW Tx Head and Tail descriptor pointers */
1011 for (i = 0; i < adapter->num_tx_queues; i++) {
1012 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1015 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1016 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1017 (tdba & DMA_BIT_MASK(32)));
1018 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1019 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1020 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1021 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1022 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1023 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1024 /* Disable Tx Head Writeback RO bit, since this hoses
1025 * bookkeeping if things aren't delivered in order.
1027 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1028 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1029 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1033 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1035 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1037 struct ixgbevf_ring *rx_ring;
1038 struct ixgbe_hw *hw = &adapter->hw;
1041 rx_ring = &adapter->rx_ring[index];
1043 srrctl = IXGBE_SRRCTL_DROP_EN;
1045 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1047 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1048 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1050 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1053 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1055 struct ixgbe_hw *hw = &adapter->hw;
1056 struct net_device *netdev = adapter->netdev;
1057 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1061 /* notify the PF of our intent to use this size of frame */
1062 ixgbevf_rlpml_set_vf(hw, max_frame);
1064 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1065 max_frame += VLAN_HLEN;
1068 * Make best use of allocation by using all but 1K of a
1069 * power of 2 allocation that will be used for skb->head.
1071 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1072 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1073 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1074 else if (max_frame <= IXGBEVF_RXBUFFER_3K)
1075 rx_buf_len = IXGBEVF_RXBUFFER_3K;
1076 else if (max_frame <= IXGBEVF_RXBUFFER_7K)
1077 rx_buf_len = IXGBEVF_RXBUFFER_7K;
1078 else if (max_frame <= IXGBEVF_RXBUFFER_15K)
1079 rx_buf_len = IXGBEVF_RXBUFFER_15K;
1081 rx_buf_len = IXGBEVF_MAX_RXBUFFER;
1083 for (i = 0; i < adapter->num_rx_queues; i++)
1084 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1088 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1089 * @adapter: board private structure
1091 * Configure the Rx unit of the MAC after a reset.
1093 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1096 struct ixgbe_hw *hw = &adapter->hw;
1100 /* PSRTYPE must be initialized in 82599 */
1101 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1103 /* set_rx_buffer_len must be called before ring initialization */
1104 ixgbevf_set_rx_buffer_len(adapter);
1106 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1107 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1108 * the Base and Length of the Rx Descriptor Ring */
1109 for (i = 0; i < adapter->num_rx_queues; i++) {
1110 rdba = adapter->rx_ring[i].dma;
1111 j = adapter->rx_ring[i].reg_idx;
1112 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1113 (rdba & DMA_BIT_MASK(32)));
1114 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1115 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1116 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1117 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1118 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1119 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1121 ixgbevf_configure_srrctl(adapter, j);
1125 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1127 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1128 struct ixgbe_hw *hw = &adapter->hw;
1131 if (!hw->mac.ops.set_vfta)
1134 spin_lock(&adapter->mbx_lock);
1136 /* add VID to filter table */
1137 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1139 spin_unlock(&adapter->mbx_lock);
1141 /* translate error return types so error makes sense */
1142 if (err == IXGBE_ERR_MBX)
1145 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1148 set_bit(vid, adapter->active_vlans);
1153 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1155 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1156 struct ixgbe_hw *hw = &adapter->hw;
1157 int err = -EOPNOTSUPP;
1159 spin_lock(&adapter->mbx_lock);
1161 /* remove VID from filter table */
1162 if (hw->mac.ops.set_vfta)
1163 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1165 spin_unlock(&adapter->mbx_lock);
1167 clear_bit(vid, adapter->active_vlans);
1172 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1176 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1177 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
1180 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1182 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1183 struct ixgbe_hw *hw = &adapter->hw;
1186 if ((netdev_uc_count(netdev)) > 10) {
1187 pr_err("Too many unicast filters - No Space\n");
1191 if (!netdev_uc_empty(netdev)) {
1192 struct netdev_hw_addr *ha;
1193 netdev_for_each_uc_addr(ha, netdev) {
1194 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1199 * If the list is empty then send message to PF driver to
1200 * clear all macvlans on this VF.
1202 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1209 * ixgbevf_set_rx_mode - Multicast set
1210 * @netdev: network interface device structure
1212 * The set_rx_method entry point is called whenever the multicast address
1213 * list or the network interface flags are updated. This routine is
1214 * responsible for configuring the hardware for proper multicast mode.
1216 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1218 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1219 struct ixgbe_hw *hw = &adapter->hw;
1221 spin_lock(&adapter->mbx_lock);
1223 /* reprogram multicast list */
1224 if (hw->mac.ops.update_mc_addr_list)
1225 hw->mac.ops.update_mc_addr_list(hw, netdev);
1227 ixgbevf_write_uc_addr_list(netdev);
1229 spin_unlock(&adapter->mbx_lock);
1232 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1235 struct ixgbevf_q_vector *q_vector;
1236 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1238 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1239 q_vector = adapter->q_vector[q_idx];
1240 napi_enable(&q_vector->napi);
1244 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1247 struct ixgbevf_q_vector *q_vector;
1248 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1250 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1251 q_vector = adapter->q_vector[q_idx];
1252 napi_disable(&q_vector->napi);
1256 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1258 struct net_device *netdev = adapter->netdev;
1261 ixgbevf_set_rx_mode(netdev);
1263 ixgbevf_restore_vlan(adapter);
1265 ixgbevf_configure_tx(adapter);
1266 ixgbevf_configure_rx(adapter);
1267 for (i = 0; i < adapter->num_rx_queues; i++) {
1268 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1269 ixgbevf_alloc_rx_buffers(adapter, ring,
1270 IXGBE_DESC_UNUSED(ring));
1274 #define IXGBE_MAX_RX_DESC_POLL 10
1275 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1278 struct ixgbe_hw *hw = &adapter->hw;
1279 int j = adapter->rx_ring[rxr].reg_idx;
1282 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1283 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1288 if (k >= IXGBE_MAX_RX_DESC_POLL) {
1289 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1290 "not set within the polling period\n", rxr);
1293 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1294 (adapter->rx_ring[rxr].count - 1));
1297 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1299 /* Only save pre-reset stats if there are some */
1300 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1301 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1302 adapter->stats.base_vfgprc;
1303 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1304 adapter->stats.base_vfgptc;
1305 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1306 adapter->stats.base_vfgorc;
1307 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1308 adapter->stats.base_vfgotc;
1309 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1310 adapter->stats.base_vfmprc;
1314 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1316 struct ixgbe_hw *hw = &adapter->hw;
1318 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1319 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1320 adapter->stats.last_vfgorc |=
1321 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1322 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1323 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1324 adapter->stats.last_vfgotc |=
1325 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1326 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1328 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1329 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1330 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1331 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1332 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1335 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1337 struct ixgbe_hw *hw = &adapter->hw;
1338 int api[] = { ixgbe_mbox_api_10,
1339 ixgbe_mbox_api_unknown };
1340 int err = 0, idx = 0;
1342 spin_lock(&adapter->mbx_lock);
1344 while (api[idx] != ixgbe_mbox_api_unknown) {
1345 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1351 spin_unlock(&adapter->mbx_lock);
1354 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1356 struct net_device *netdev = adapter->netdev;
1357 struct ixgbe_hw *hw = &adapter->hw;
1359 int num_rx_rings = adapter->num_rx_queues;
1362 for (i = 0; i < adapter->num_tx_queues; i++) {
1363 j = adapter->tx_ring[i].reg_idx;
1364 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1365 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1366 txdctl |= (8 << 16);
1367 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1370 for (i = 0; i < adapter->num_tx_queues; i++) {
1371 j = adapter->tx_ring[i].reg_idx;
1372 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1373 txdctl |= IXGBE_TXDCTL_ENABLE;
1374 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1377 for (i = 0; i < num_rx_rings; i++) {
1378 j = adapter->rx_ring[i].reg_idx;
1379 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1380 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1381 if (hw->mac.type == ixgbe_mac_X540_vf) {
1382 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1383 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1384 IXGBE_RXDCTL_RLPML_EN);
1386 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1387 ixgbevf_rx_desc_queue_enable(adapter, i);
1390 ixgbevf_configure_msix(adapter);
1392 spin_lock(&adapter->mbx_lock);
1394 if (hw->mac.ops.set_rar) {
1395 if (is_valid_ether_addr(hw->mac.addr))
1396 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1398 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1401 spin_unlock(&adapter->mbx_lock);
1403 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1404 ixgbevf_napi_enable_all(adapter);
1406 /* enable transmits */
1407 netif_tx_start_all_queues(netdev);
1409 ixgbevf_save_reset_stats(adapter);
1410 ixgbevf_init_last_counter_stats(adapter);
1412 hw->mac.get_link_status = 1;
1413 mod_timer(&adapter->watchdog_timer, jiffies);
1416 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1418 struct ixgbe_hw *hw = &adapter->hw;
1420 ixgbevf_negotiate_api(adapter);
1422 ixgbevf_configure(adapter);
1424 ixgbevf_up_complete(adapter);
1426 /* clear any pending interrupts, may auto mask */
1427 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1429 ixgbevf_irq_enable(adapter);
1433 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1434 * @adapter: board private structure
1435 * @rx_ring: ring to free buffers from
1437 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1438 struct ixgbevf_ring *rx_ring)
1440 struct pci_dev *pdev = adapter->pdev;
1444 if (!rx_ring->rx_buffer_info)
1447 /* Free all the Rx ring sk_buffs */
1448 for (i = 0; i < rx_ring->count; i++) {
1449 struct ixgbevf_rx_buffer *rx_buffer_info;
1451 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1452 if (rx_buffer_info->dma) {
1453 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1454 rx_ring->rx_buf_len,
1456 rx_buffer_info->dma = 0;
1458 if (rx_buffer_info->skb) {
1459 struct sk_buff *skb = rx_buffer_info->skb;
1460 rx_buffer_info->skb = NULL;
1462 struct sk_buff *this = skb;
1463 skb = IXGBE_CB(skb)->prev;
1464 dev_kfree_skb(this);
1469 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1470 memset(rx_ring->rx_buffer_info, 0, size);
1472 /* Zero out the descriptor ring */
1473 memset(rx_ring->desc, 0, rx_ring->size);
1475 rx_ring->next_to_clean = 0;
1476 rx_ring->next_to_use = 0;
1479 writel(0, adapter->hw.hw_addr + rx_ring->head);
1481 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1485 * ixgbevf_clean_tx_ring - Free Tx Buffers
1486 * @adapter: board private structure
1487 * @tx_ring: ring to be cleaned
1489 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1490 struct ixgbevf_ring *tx_ring)
1492 struct ixgbevf_tx_buffer *tx_buffer_info;
1496 if (!tx_ring->tx_buffer_info)
1499 /* Free all the Tx ring sk_buffs */
1501 for (i = 0; i < tx_ring->count; i++) {
1502 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1503 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1506 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1507 memset(tx_ring->tx_buffer_info, 0, size);
1509 memset(tx_ring->desc, 0, tx_ring->size);
1511 tx_ring->next_to_use = 0;
1512 tx_ring->next_to_clean = 0;
1515 writel(0, adapter->hw.hw_addr + tx_ring->head);
1517 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1521 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1522 * @adapter: board private structure
1524 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1528 for (i = 0; i < adapter->num_rx_queues; i++)
1529 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1533 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1534 * @adapter: board private structure
1536 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1540 for (i = 0; i < adapter->num_tx_queues; i++)
1541 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1544 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1546 struct net_device *netdev = adapter->netdev;
1547 struct ixgbe_hw *hw = &adapter->hw;
1551 /* signal that we are down to the interrupt handler */
1552 set_bit(__IXGBEVF_DOWN, &adapter->state);
1553 /* disable receives */
1555 netif_tx_disable(netdev);
1559 netif_tx_stop_all_queues(netdev);
1561 ixgbevf_irq_disable(adapter);
1563 ixgbevf_napi_disable_all(adapter);
1565 del_timer_sync(&adapter->watchdog_timer);
1566 /* can't call flush scheduled work here because it can deadlock
1567 * if linkwatch_event tries to acquire the rtnl_lock which we are
1569 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1572 /* disable transmits in the hardware now that interrupts are off */
1573 for (i = 0; i < adapter->num_tx_queues; i++) {
1574 j = adapter->tx_ring[i].reg_idx;
1575 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1576 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1577 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1580 netif_carrier_off(netdev);
1582 if (!pci_channel_offline(adapter->pdev))
1583 ixgbevf_reset(adapter);
1585 ixgbevf_clean_all_tx_rings(adapter);
1586 ixgbevf_clean_all_rx_rings(adapter);
1589 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1591 WARN_ON(in_interrupt());
1593 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1597 * Check if PF is up before re-init. If not then skip until
1598 * later when the PF is up and ready to service requests from
1599 * the VF via mailbox. If the VF is up and running then the
1600 * watchdog task will continue to schedule reset tasks until
1601 * the PF is up and running.
1603 ixgbevf_down(adapter);
1604 ixgbevf_up(adapter);
1606 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1609 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1611 struct ixgbe_hw *hw = &adapter->hw;
1612 struct net_device *netdev = adapter->netdev;
1614 spin_lock(&adapter->mbx_lock);
1616 if (hw->mac.ops.reset_hw(hw))
1617 hw_dbg(hw, "PF still resetting\n");
1619 hw->mac.ops.init_hw(hw);
1621 spin_unlock(&adapter->mbx_lock);
1623 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1624 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1626 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1631 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1634 int err, vector_threshold;
1636 /* We'll want at least 2 (vector_threshold):
1637 * 1) TxQ[0] + RxQ[0] handler
1638 * 2) Other (Link Status Change, etc.)
1640 vector_threshold = MIN_MSIX_COUNT;
1642 /* The more we get, the more we will assign to Tx/Rx Cleanup
1643 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1644 * Right now, we simply care about how many we'll get; we'll
1645 * set them up later while requesting irq's.
1647 while (vectors >= vector_threshold) {
1648 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1650 if (!err) /* Success in acquiring all requested vectors. */
1653 vectors = 0; /* Nasty failure, quit now */
1654 else /* err == number of vectors we should try again with */
1658 if (vectors < vector_threshold) {
1659 /* Can't allocate enough MSI-X interrupts? Oh well.
1660 * This just means we'll go with either a single MSI
1661 * vector or fall back to legacy interrupts.
1663 hw_dbg(&adapter->hw,
1664 "Unable to allocate MSI-X interrupts\n");
1665 kfree(adapter->msix_entries);
1666 adapter->msix_entries = NULL;
1669 * Adjust for only the vectors we'll use, which is minimum
1670 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1671 * vectors we were allocated.
1673 adapter->num_msix_vectors = vectors;
1678 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1679 * @adapter: board private structure to initialize
1681 * This is the top level queue allocation routine. The order here is very
1682 * important, starting with the "most" number of features turned on at once,
1683 * and ending with the smallest set of features. This way large combinations
1684 * can be allocated if they're turned on, and smaller combinations are the
1685 * fallthrough conditions.
1688 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1690 /* Start with base case */
1691 adapter->num_rx_queues = 1;
1692 adapter->num_tx_queues = 1;
1696 * ixgbevf_alloc_queues - Allocate memory for all rings
1697 * @adapter: board private structure to initialize
1699 * We allocate one ring per queue at run-time since we don't know the
1700 * number of queues at compile-time. The polling_netdev array is
1701 * intended for Multiqueue, but should work fine with a single queue.
1703 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1707 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1708 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1709 if (!adapter->tx_ring)
1710 goto err_tx_ring_allocation;
1712 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1713 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1714 if (!adapter->rx_ring)
1715 goto err_rx_ring_allocation;
1717 for (i = 0; i < adapter->num_tx_queues; i++) {
1718 adapter->tx_ring[i].count = adapter->tx_ring_count;
1719 adapter->tx_ring[i].queue_index = i;
1720 adapter->tx_ring[i].reg_idx = i;
1721 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1722 adapter->tx_ring[i].netdev = adapter->netdev;
1725 for (i = 0; i < adapter->num_rx_queues; i++) {
1726 adapter->rx_ring[i].count = adapter->rx_ring_count;
1727 adapter->rx_ring[i].queue_index = i;
1728 adapter->rx_ring[i].reg_idx = i;
1729 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1730 adapter->rx_ring[i].netdev = adapter->netdev;
1735 err_rx_ring_allocation:
1736 kfree(adapter->tx_ring);
1737 err_tx_ring_allocation:
1742 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1743 * @adapter: board private structure to initialize
1745 * Attempt to configure the interrupts using the best available
1746 * capabilities of the hardware and the kernel.
1748 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1750 struct net_device *netdev = adapter->netdev;
1752 int vector, v_budget;
1755 * It's easy to be greedy for MSI-X vectors, but it really
1756 * doesn't do us much good if we have a lot more vectors
1757 * than CPU's. So let's be conservative and only ask for
1758 * (roughly) the same number of vectors as there are CPU's.
1759 * The default is to use pairs of vectors.
1761 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1762 v_budget = min_t(int, v_budget, num_online_cpus());
1763 v_budget += NON_Q_VECTORS;
1765 /* A failure in MSI-X entry allocation isn't fatal, but it does
1766 * mean we disable MSI-X capabilities of the adapter. */
1767 adapter->msix_entries = kcalloc(v_budget,
1768 sizeof(struct msix_entry), GFP_KERNEL);
1769 if (!adapter->msix_entries) {
1774 for (vector = 0; vector < v_budget; vector++)
1775 adapter->msix_entries[vector].entry = vector;
1777 ixgbevf_acquire_msix_vectors(adapter, v_budget);
1779 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1783 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1790 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1791 * @adapter: board private structure to initialize
1793 * We allocate one q_vector per queue interrupt. If allocation fails we
1796 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1798 int q_idx, num_q_vectors;
1799 struct ixgbevf_q_vector *q_vector;
1801 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1803 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1804 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1807 q_vector->adapter = adapter;
1808 q_vector->v_idx = q_idx;
1809 netif_napi_add(adapter->netdev, &q_vector->napi,
1811 adapter->q_vector[q_idx] = q_vector;
1819 q_vector = adapter->q_vector[q_idx];
1820 netif_napi_del(&q_vector->napi);
1822 adapter->q_vector[q_idx] = NULL;
1828 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1829 * @adapter: board private structure to initialize
1831 * This function frees the memory allocated to the q_vectors. In addition if
1832 * NAPI is enabled it will delete any references to the NAPI struct prior
1833 * to freeing the q_vector.
1835 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1837 int q_idx, num_q_vectors;
1840 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1841 napi_vectors = adapter->num_rx_queues;
1843 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1844 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1846 adapter->q_vector[q_idx] = NULL;
1847 if (q_idx < napi_vectors)
1848 netif_napi_del(&q_vector->napi);
1854 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1855 * @adapter: board private structure
1858 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1860 pci_disable_msix(adapter->pdev);
1861 kfree(adapter->msix_entries);
1862 adapter->msix_entries = NULL;
1866 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1867 * @adapter: board private structure to initialize
1870 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1874 /* Number of supported queues */
1875 ixgbevf_set_num_queues(adapter);
1877 err = ixgbevf_set_interrupt_capability(adapter);
1879 hw_dbg(&adapter->hw,
1880 "Unable to setup interrupt capabilities\n");
1881 goto err_set_interrupt;
1884 err = ixgbevf_alloc_q_vectors(adapter);
1886 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
1888 goto err_alloc_q_vectors;
1891 err = ixgbevf_alloc_queues(adapter);
1893 pr_err("Unable to allocate memory for queues\n");
1894 goto err_alloc_queues;
1897 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
1898 "Tx Queue count = %u\n",
1899 (adapter->num_rx_queues > 1) ? "Enabled" :
1900 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
1902 set_bit(__IXGBEVF_DOWN, &adapter->state);
1906 ixgbevf_free_q_vectors(adapter);
1907 err_alloc_q_vectors:
1908 ixgbevf_reset_interrupt_capability(adapter);
1914 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
1915 * @adapter: board private structure to clear interrupt scheme on
1917 * We go through and clear interrupt specific resources and reset the structure
1918 * to pre-load conditions
1920 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
1922 adapter->num_tx_queues = 0;
1923 adapter->num_rx_queues = 0;
1925 ixgbevf_free_q_vectors(adapter);
1926 ixgbevf_reset_interrupt_capability(adapter);
1930 * ixgbevf_sw_init - Initialize general software structures
1931 * (struct ixgbevf_adapter)
1932 * @adapter: board private structure to initialize
1934 * ixgbevf_sw_init initializes the Adapter private data structure.
1935 * Fields are initialized based on PCI device information and
1936 * OS network device settings (MTU size).
1938 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
1940 struct ixgbe_hw *hw = &adapter->hw;
1941 struct pci_dev *pdev = adapter->pdev;
1944 /* PCI config space info */
1946 hw->vendor_id = pdev->vendor;
1947 hw->device_id = pdev->device;
1948 hw->revision_id = pdev->revision;
1949 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1950 hw->subsystem_device_id = pdev->subsystem_device;
1952 hw->mbx.ops.init_params(hw);
1953 hw->mac.max_tx_queues = MAX_TX_QUEUES;
1954 hw->mac.max_rx_queues = MAX_RX_QUEUES;
1955 err = hw->mac.ops.reset_hw(hw);
1957 dev_info(&pdev->dev,
1958 "PF still in reset state, assigning new address\n");
1959 eth_hw_addr_random(adapter->netdev);
1960 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
1961 adapter->netdev->addr_len);
1963 err = hw->mac.ops.init_hw(hw);
1965 pr_err("init_shared_code failed: %d\n", err);
1968 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
1969 adapter->netdev->addr_len);
1972 /* lock to protect mailbox accesses */
1973 spin_lock_init(&adapter->mbx_lock);
1975 /* Enable dynamic interrupt throttling rates */
1976 adapter->rx_itr_setting = 1;
1977 adapter->tx_itr_setting = 1;
1979 /* set default ring sizes */
1980 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
1981 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
1983 set_bit(__IXGBEVF_DOWN, &adapter->state);
1990 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
1992 u32 current_counter = IXGBE_READ_REG(hw, reg); \
1993 if (current_counter < last_counter) \
1994 counter += 0x100000000LL; \
1995 last_counter = current_counter; \
1996 counter &= 0xFFFFFFFF00000000LL; \
1997 counter |= current_counter; \
2000 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2002 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2003 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2004 u64 current_counter = (current_counter_msb << 32) | \
2005 current_counter_lsb; \
2006 if (current_counter < last_counter) \
2007 counter += 0x1000000000LL; \
2008 last_counter = current_counter; \
2009 counter &= 0xFFFFFFF000000000LL; \
2010 counter |= current_counter; \
2013 * ixgbevf_update_stats - Update the board statistics counters.
2014 * @adapter: board private structure
2016 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2018 struct ixgbe_hw *hw = &adapter->hw;
2020 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2021 adapter->stats.vfgprc);
2022 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2023 adapter->stats.vfgptc);
2024 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2025 adapter->stats.last_vfgorc,
2026 adapter->stats.vfgorc);
2027 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2028 adapter->stats.last_vfgotc,
2029 adapter->stats.vfgotc);
2030 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2031 adapter->stats.vfmprc);
2035 * ixgbevf_watchdog - Timer Call-back
2036 * @data: pointer to adapter cast into an unsigned long
2038 static void ixgbevf_watchdog(unsigned long data)
2040 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2041 struct ixgbe_hw *hw = &adapter->hw;
2046 * Do the watchdog outside of interrupt context due to the lovely
2047 * delays that some of the newer hardware requires
2050 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2051 goto watchdog_short_circuit;
2053 /* get one bit for every active tx/rx interrupt vector */
2054 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2055 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2056 if (qv->rx.ring || qv->tx.ring)
2060 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2062 watchdog_short_circuit:
2063 schedule_work(&adapter->watchdog_task);
2067 * ixgbevf_tx_timeout - Respond to a Tx Hang
2068 * @netdev: network interface device structure
2070 static void ixgbevf_tx_timeout(struct net_device *netdev)
2072 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2074 /* Do the reset outside of interrupt context */
2075 schedule_work(&adapter->reset_task);
2078 static void ixgbevf_reset_task(struct work_struct *work)
2080 struct ixgbevf_adapter *adapter;
2081 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2083 /* If we're already down or resetting, just bail */
2084 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2085 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2088 adapter->tx_timeout_count++;
2090 ixgbevf_reinit_locked(adapter);
2094 * ixgbevf_watchdog_task - worker thread to bring link up
2095 * @work: pointer to work_struct containing our data
2097 static void ixgbevf_watchdog_task(struct work_struct *work)
2099 struct ixgbevf_adapter *adapter = container_of(work,
2100 struct ixgbevf_adapter,
2102 struct net_device *netdev = adapter->netdev;
2103 struct ixgbe_hw *hw = &adapter->hw;
2104 u32 link_speed = adapter->link_speed;
2105 bool link_up = adapter->link_up;
2107 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2110 * Always check the link on the watchdog because we have
2113 if (hw->mac.ops.check_link) {
2116 spin_lock(&adapter->mbx_lock);
2118 need_reset = hw->mac.ops.check_link(hw, &link_speed,
2121 spin_unlock(&adapter->mbx_lock);
2124 adapter->link_up = link_up;
2125 adapter->link_speed = link_speed;
2126 netif_carrier_off(netdev);
2127 netif_tx_stop_all_queues(netdev);
2128 schedule_work(&adapter->reset_task);
2132 /* always assume link is up, if no check link
2134 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2137 adapter->link_up = link_up;
2138 adapter->link_speed = link_speed;
2141 if (!netif_carrier_ok(netdev)) {
2142 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2143 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2145 netif_carrier_on(netdev);
2146 netif_tx_wake_all_queues(netdev);
2149 adapter->link_up = false;
2150 adapter->link_speed = 0;
2151 if (netif_carrier_ok(netdev)) {
2152 hw_dbg(&adapter->hw, "NIC Link is Down\n");
2153 netif_carrier_off(netdev);
2154 netif_tx_stop_all_queues(netdev);
2158 ixgbevf_update_stats(adapter);
2161 /* Reset the timer */
2162 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2163 mod_timer(&adapter->watchdog_timer,
2164 round_jiffies(jiffies + (2 * HZ)));
2166 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2170 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2171 * @adapter: board private structure
2172 * @tx_ring: Tx descriptor ring for a specific queue
2174 * Free all transmit software resources
2176 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2177 struct ixgbevf_ring *tx_ring)
2179 struct pci_dev *pdev = adapter->pdev;
2181 ixgbevf_clean_tx_ring(adapter, tx_ring);
2183 vfree(tx_ring->tx_buffer_info);
2184 tx_ring->tx_buffer_info = NULL;
2186 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2189 tx_ring->desc = NULL;
2193 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2194 * @adapter: board private structure
2196 * Free all transmit software resources
2198 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2202 for (i = 0; i < adapter->num_tx_queues; i++)
2203 if (adapter->tx_ring[i].desc)
2204 ixgbevf_free_tx_resources(adapter,
2205 &adapter->tx_ring[i]);
2210 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2211 * @adapter: board private structure
2212 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2214 * Return 0 on success, negative on failure
2216 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2217 struct ixgbevf_ring *tx_ring)
2219 struct pci_dev *pdev = adapter->pdev;
2222 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2223 tx_ring->tx_buffer_info = vzalloc(size);
2224 if (!tx_ring->tx_buffer_info)
2227 /* round up to nearest 4K */
2228 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2229 tx_ring->size = ALIGN(tx_ring->size, 4096);
2231 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2232 &tx_ring->dma, GFP_KERNEL);
2236 tx_ring->next_to_use = 0;
2237 tx_ring->next_to_clean = 0;
2241 vfree(tx_ring->tx_buffer_info);
2242 tx_ring->tx_buffer_info = NULL;
2243 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2244 "descriptor ring\n");
2249 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2250 * @adapter: board private structure
2252 * If this function returns with an error, then it's possible one or
2253 * more of the rings is populated (while the rest are not). It is the
2254 * callers duty to clean those orphaned rings.
2256 * Return 0 on success, negative on failure
2258 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2262 for (i = 0; i < adapter->num_tx_queues; i++) {
2263 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2266 hw_dbg(&adapter->hw,
2267 "Allocation for Tx Queue %u failed\n", i);
2275 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2276 * @adapter: board private structure
2277 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2279 * Returns 0 on success, negative on failure
2281 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2282 struct ixgbevf_ring *rx_ring)
2284 struct pci_dev *pdev = adapter->pdev;
2287 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2288 rx_ring->rx_buffer_info = vzalloc(size);
2289 if (!rx_ring->rx_buffer_info)
2292 /* Round up to nearest 4K */
2293 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2294 rx_ring->size = ALIGN(rx_ring->size, 4096);
2296 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2297 &rx_ring->dma, GFP_KERNEL);
2299 if (!rx_ring->desc) {
2300 hw_dbg(&adapter->hw,
2301 "Unable to allocate memory for "
2302 "the receive descriptor ring\n");
2303 vfree(rx_ring->rx_buffer_info);
2304 rx_ring->rx_buffer_info = NULL;
2308 rx_ring->next_to_clean = 0;
2309 rx_ring->next_to_use = 0;
2317 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2318 * @adapter: board private structure
2320 * If this function returns with an error, then it's possible one or
2321 * more of the rings is populated (while the rest are not). It is the
2322 * callers duty to clean those orphaned rings.
2324 * Return 0 on success, negative on failure
2326 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2330 for (i = 0; i < adapter->num_rx_queues; i++) {
2331 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2334 hw_dbg(&adapter->hw,
2335 "Allocation for Rx Queue %u failed\n", i);
2342 * ixgbevf_free_rx_resources - Free Rx Resources
2343 * @adapter: board private structure
2344 * @rx_ring: ring to clean the resources from
2346 * Free all receive software resources
2348 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2349 struct ixgbevf_ring *rx_ring)
2351 struct pci_dev *pdev = adapter->pdev;
2353 ixgbevf_clean_rx_ring(adapter, rx_ring);
2355 vfree(rx_ring->rx_buffer_info);
2356 rx_ring->rx_buffer_info = NULL;
2358 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2361 rx_ring->desc = NULL;
2365 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2366 * @adapter: board private structure
2368 * Free all receive software resources
2370 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2374 for (i = 0; i < adapter->num_rx_queues; i++)
2375 if (adapter->rx_ring[i].desc)
2376 ixgbevf_free_rx_resources(adapter,
2377 &adapter->rx_ring[i]);
2381 * ixgbevf_open - Called when a network interface is made active
2382 * @netdev: network interface device structure
2384 * Returns 0 on success, negative value on failure
2386 * The open entry point is called when a network interface is made
2387 * active by the system (IFF_UP). At this point all resources needed
2388 * for transmit and receive operations are allocated, the interrupt
2389 * handler is registered with the OS, the watchdog timer is started,
2390 * and the stack is notified that the interface is ready.
2392 static int ixgbevf_open(struct net_device *netdev)
2394 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2395 struct ixgbe_hw *hw = &adapter->hw;
2398 /* disallow open during test */
2399 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2402 if (hw->adapter_stopped) {
2403 ixgbevf_reset(adapter);
2404 /* if adapter is still stopped then PF isn't up and
2405 * the vf can't start. */
2406 if (hw->adapter_stopped) {
2407 err = IXGBE_ERR_MBX;
2408 pr_err("Unable to start - perhaps the PF Driver isn't "
2410 goto err_setup_reset;
2414 ixgbevf_negotiate_api(adapter);
2416 /* allocate transmit descriptors */
2417 err = ixgbevf_setup_all_tx_resources(adapter);
2421 /* allocate receive descriptors */
2422 err = ixgbevf_setup_all_rx_resources(adapter);
2426 ixgbevf_configure(adapter);
2429 * Map the Tx/Rx rings to the vectors we were allotted.
2430 * if request_irq will be called in this function map_rings
2431 * must be called *before* up_complete
2433 ixgbevf_map_rings_to_vectors(adapter);
2435 ixgbevf_up_complete(adapter);
2437 /* clear any pending interrupts, may auto mask */
2438 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2439 err = ixgbevf_request_irq(adapter);
2443 ixgbevf_irq_enable(adapter);
2448 ixgbevf_down(adapter);
2449 ixgbevf_free_irq(adapter);
2451 ixgbevf_free_all_rx_resources(adapter);
2453 ixgbevf_free_all_tx_resources(adapter);
2454 ixgbevf_reset(adapter);
2462 * ixgbevf_close - Disables a network interface
2463 * @netdev: network interface device structure
2465 * Returns 0, this is not allowed to fail
2467 * The close entry point is called when an interface is de-activated
2468 * by the OS. The hardware is still under the drivers control, but
2469 * needs to be disabled. A global MAC reset is issued to stop the
2470 * hardware, and all transmit and receive resources are freed.
2472 static int ixgbevf_close(struct net_device *netdev)
2474 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2476 ixgbevf_down(adapter);
2477 ixgbevf_free_irq(adapter);
2479 ixgbevf_free_all_tx_resources(adapter);
2480 ixgbevf_free_all_rx_resources(adapter);
2485 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2486 u32 vlan_macip_lens, u32 type_tucmd,
2489 struct ixgbe_adv_tx_context_desc *context_desc;
2490 u16 i = tx_ring->next_to_use;
2492 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2495 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2497 /* set bits to identify this as an advanced context descriptor */
2498 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2500 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2501 context_desc->seqnum_seed = 0;
2502 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2503 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2506 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2507 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2509 u32 vlan_macip_lens, type_tucmd;
2510 u32 mss_l4len_idx, l4len;
2512 if (!skb_is_gso(skb))
2515 if (skb_header_cloned(skb)) {
2516 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2521 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2522 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2524 if (skb->protocol == htons(ETH_P_IP)) {
2525 struct iphdr *iph = ip_hdr(skb);
2528 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2532 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2533 } else if (skb_is_gso_v6(skb)) {
2534 ipv6_hdr(skb)->payload_len = 0;
2535 tcp_hdr(skb)->check =
2536 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2537 &ipv6_hdr(skb)->daddr,
2541 /* compute header lengths */
2542 l4len = tcp_hdrlen(skb);
2544 *hdr_len = skb_transport_offset(skb) + l4len;
2546 /* mss_l4len_id: use 1 as index for TSO */
2547 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2548 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2549 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2551 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2552 vlan_macip_lens = skb_network_header_len(skb);
2553 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2554 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2556 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2557 type_tucmd, mss_l4len_idx);
2562 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2563 struct sk_buff *skb, u32 tx_flags)
2568 u32 vlan_macip_lens = 0;
2569 u32 mss_l4len_idx = 0;
2572 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2574 switch (skb->protocol) {
2575 case __constant_htons(ETH_P_IP):
2576 vlan_macip_lens |= skb_network_header_len(skb);
2577 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2578 l4_hdr = ip_hdr(skb)->protocol;
2580 case __constant_htons(ETH_P_IPV6):
2581 vlan_macip_lens |= skb_network_header_len(skb);
2582 l4_hdr = ipv6_hdr(skb)->nexthdr;
2585 if (unlikely(net_ratelimit())) {
2586 dev_warn(tx_ring->dev,
2587 "partial checksum but proto=%x!\n",
2595 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2596 mss_l4len_idx = tcp_hdrlen(skb) <<
2597 IXGBE_ADVTXD_L4LEN_SHIFT;
2600 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2601 mss_l4len_idx = sizeof(struct sctphdr) <<
2602 IXGBE_ADVTXD_L4LEN_SHIFT;
2605 mss_l4len_idx = sizeof(struct udphdr) <<
2606 IXGBE_ADVTXD_L4LEN_SHIFT;
2609 if (unlikely(net_ratelimit())) {
2610 dev_warn(tx_ring->dev,
2611 "partial checksum but l4 proto=%x!\n",
2618 /* vlan_macip_lens: MACLEN, VLAN tag */
2619 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2620 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2622 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2623 type_tucmd, mss_l4len_idx);
2625 return (skb->ip_summed == CHECKSUM_PARTIAL);
2628 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2629 struct sk_buff *skb, u32 tx_flags,
2632 struct ixgbevf_tx_buffer *tx_buffer_info;
2634 unsigned int total = skb->len;
2635 unsigned int offset = 0, size;
2637 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2641 i = tx_ring->next_to_use;
2643 len = min(skb_headlen(skb), total);
2645 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2646 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2648 tx_buffer_info->length = size;
2649 tx_buffer_info->mapped_as_page = false;
2650 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
2652 size, DMA_TO_DEVICE);
2653 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
2655 tx_buffer_info->next_to_watch = i;
2662 if (i == tx_ring->count)
2666 for (f = 0; f < nr_frags; f++) {
2667 const struct skb_frag_struct *frag;
2669 frag = &skb_shinfo(skb)->frags[f];
2670 len = min((unsigned int)skb_frag_size(frag), total);
2674 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2675 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2677 tx_buffer_info->length = size;
2678 tx_buffer_info->dma =
2679 skb_frag_dma_map(tx_ring->dev, frag,
2680 offset, size, DMA_TO_DEVICE);
2681 tx_buffer_info->mapped_as_page = true;
2682 if (dma_mapping_error(tx_ring->dev,
2683 tx_buffer_info->dma))
2685 tx_buffer_info->next_to_watch = i;
2692 if (i == tx_ring->count)
2700 i = tx_ring->count - 1;
2703 tx_ring->tx_buffer_info[i].skb = skb;
2704 tx_ring->tx_buffer_info[first].next_to_watch = i;
2705 tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2710 dev_err(tx_ring->dev, "TX DMA map failed\n");
2712 /* clear timestamp and dma mappings for failed tx_buffer_info map */
2713 tx_buffer_info->dma = 0;
2714 tx_buffer_info->next_to_watch = 0;
2717 /* clear timestamp and dma mappings for remaining portion of packet */
2718 while (count >= 0) {
2722 i += tx_ring->count;
2723 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2724 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2730 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
2731 int count, u32 paylen, u8 hdr_len)
2733 union ixgbe_adv_tx_desc *tx_desc = NULL;
2734 struct ixgbevf_tx_buffer *tx_buffer_info;
2735 u32 olinfo_status = 0, cmd_type_len = 0;
2738 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2740 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2742 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2744 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2745 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2747 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2748 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2750 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2751 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2753 /* use index 1 context for tso */
2754 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2755 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2756 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
2761 * Check Context must be set if Tx switch is enabled, which it
2762 * always is for case where virtual functions are running
2764 olinfo_status |= IXGBE_ADVTXD_CC;
2766 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2768 i = tx_ring->next_to_use;
2770 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2771 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2772 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2773 tx_desc->read.cmd_type_len =
2774 cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2775 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2777 if (i == tx_ring->count)
2781 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2783 tx_ring->next_to_use = i;
2786 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2788 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
2790 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2791 /* Herbert's original patch had:
2792 * smp_mb__after_netif_stop_queue();
2793 * but since that doesn't exist yet, just open code it. */
2796 /* We need to check again in a case another CPU has just
2797 * made room available. */
2798 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
2801 /* A reprieve! - use start_queue because it doesn't call schedule */
2802 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2803 ++adapter->restart_queue;
2807 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2809 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
2811 return __ixgbevf_maybe_stop_tx(tx_ring, size);
2814 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2816 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2817 struct ixgbevf_ring *tx_ring;
2819 unsigned int tx_flags = 0;
2822 u16 count = TXD_USE_COUNT(skb_headlen(skb));
2823 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2827 tx_ring = &adapter->tx_ring[r_idx];
2830 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
2831 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
2832 * + 2 desc gap to keep tail from touching head,
2833 * + 1 desc for context descriptor,
2834 * otherwise try next time
2836 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2837 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2838 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2840 count += skb_shinfo(skb)->nr_frags;
2842 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
2844 return NETDEV_TX_BUSY;
2847 if (vlan_tx_tag_present(skb)) {
2848 tx_flags |= vlan_tx_tag_get(skb);
2849 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
2850 tx_flags |= IXGBE_TX_FLAGS_VLAN;
2853 first = tx_ring->next_to_use;
2855 if (skb->protocol == htons(ETH_P_IP))
2856 tx_flags |= IXGBE_TX_FLAGS_IPV4;
2857 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
2859 dev_kfree_skb_any(skb);
2860 return NETDEV_TX_OK;
2864 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
2865 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
2866 tx_flags |= IXGBE_TX_FLAGS_CSUM;
2868 ixgbevf_tx_queue(tx_ring, tx_flags,
2869 ixgbevf_tx_map(tx_ring, skb, tx_flags, first),
2872 * Force memory writes to complete before letting h/w
2873 * know there are new descriptors to fetch. (Only
2874 * applicable for weak-ordered memory model archs,
2879 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
2881 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
2883 return NETDEV_TX_OK;
2887 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
2888 * @netdev: network interface device structure
2889 * @p: pointer to an address structure
2891 * Returns 0 on success, negative on failure
2893 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
2895 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2896 struct ixgbe_hw *hw = &adapter->hw;
2897 struct sockaddr *addr = p;
2899 if (!is_valid_ether_addr(addr->sa_data))
2900 return -EADDRNOTAVAIL;
2902 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2903 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2905 spin_lock(&adapter->mbx_lock);
2907 if (hw->mac.ops.set_rar)
2908 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2910 spin_unlock(&adapter->mbx_lock);
2916 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
2917 * @netdev: network interface device structure
2918 * @new_mtu: new value for maximum frame size
2920 * Returns 0 on success, negative on failure
2922 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
2924 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2925 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2926 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
2928 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
2929 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
2931 /* MTU < 68 is an error and causes problems on some kernels */
2932 if ((new_mtu < 68) || (max_frame > max_possible_frame))
2935 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
2936 netdev->mtu, new_mtu);
2937 /* must set new MTU before calling down or up */
2938 netdev->mtu = new_mtu;
2940 if (netif_running(netdev))
2941 ixgbevf_reinit_locked(adapter);
2946 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
2948 struct net_device *netdev = pci_get_drvdata(pdev);
2949 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2954 netif_device_detach(netdev);
2956 if (netif_running(netdev)) {
2958 ixgbevf_down(adapter);
2959 ixgbevf_free_irq(adapter);
2960 ixgbevf_free_all_tx_resources(adapter);
2961 ixgbevf_free_all_rx_resources(adapter);
2965 ixgbevf_clear_interrupt_scheme(adapter);
2968 retval = pci_save_state(pdev);
2973 pci_disable_device(pdev);
2979 static int ixgbevf_resume(struct pci_dev *pdev)
2981 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev);
2982 struct net_device *netdev = adapter->netdev;
2985 pci_set_power_state(pdev, PCI_D0);
2986 pci_restore_state(pdev);
2988 * pci_restore_state clears dev->state_saved so call
2989 * pci_save_state to restore it.
2991 pci_save_state(pdev);
2993 err = pci_enable_device_mem(pdev);
2995 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
2998 pci_set_master(pdev);
3001 err = ixgbevf_init_interrupt_scheme(adapter);
3004 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3008 ixgbevf_reset(adapter);
3010 if (netif_running(netdev)) {
3011 err = ixgbevf_open(netdev);
3016 netif_device_attach(netdev);
3021 #endif /* CONFIG_PM */
3022 static void ixgbevf_shutdown(struct pci_dev *pdev)
3024 ixgbevf_suspend(pdev, PMSG_SUSPEND);
3027 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3028 struct rtnl_link_stats64 *stats)
3030 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3033 const struct ixgbevf_ring *ring;
3036 ixgbevf_update_stats(adapter);
3038 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3040 for (i = 0; i < adapter->num_rx_queues; i++) {
3041 ring = &adapter->rx_ring[i];
3043 start = u64_stats_fetch_begin_bh(&ring->syncp);
3044 bytes = ring->total_bytes;
3045 packets = ring->total_packets;
3046 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3047 stats->rx_bytes += bytes;
3048 stats->rx_packets += packets;
3051 for (i = 0; i < adapter->num_tx_queues; i++) {
3052 ring = &adapter->tx_ring[i];
3054 start = u64_stats_fetch_begin_bh(&ring->syncp);
3055 bytes = ring->total_bytes;
3056 packets = ring->total_packets;
3057 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3058 stats->tx_bytes += bytes;
3059 stats->tx_packets += packets;
3065 static const struct net_device_ops ixgbevf_netdev_ops = {
3066 .ndo_open = ixgbevf_open,
3067 .ndo_stop = ixgbevf_close,
3068 .ndo_start_xmit = ixgbevf_xmit_frame,
3069 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
3070 .ndo_get_stats64 = ixgbevf_get_stats,
3071 .ndo_validate_addr = eth_validate_addr,
3072 .ndo_set_mac_address = ixgbevf_set_mac,
3073 .ndo_change_mtu = ixgbevf_change_mtu,
3074 .ndo_tx_timeout = ixgbevf_tx_timeout,
3075 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3076 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
3079 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3081 dev->netdev_ops = &ixgbevf_netdev_ops;
3082 ixgbevf_set_ethtool_ops(dev);
3083 dev->watchdog_timeo = 5 * HZ;
3087 * ixgbevf_probe - Device Initialization Routine
3088 * @pdev: PCI device information struct
3089 * @ent: entry in ixgbevf_pci_tbl
3091 * Returns 0 on success, negative on failure
3093 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3094 * The OS initialization, configuring of the adapter private structure,
3095 * and a hardware reset occur.
3097 static int __devinit ixgbevf_probe(struct pci_dev *pdev,
3098 const struct pci_device_id *ent)
3100 struct net_device *netdev;
3101 struct ixgbevf_adapter *adapter = NULL;
3102 struct ixgbe_hw *hw = NULL;
3103 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3104 static int cards_found;
3105 int err, pci_using_dac;
3107 err = pci_enable_device(pdev);
3111 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3112 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
3115 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
3117 err = dma_set_coherent_mask(&pdev->dev,
3120 dev_err(&pdev->dev, "No usable DMA "
3121 "configuration, aborting\n");
3128 err = pci_request_regions(pdev, ixgbevf_driver_name);
3130 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3134 pci_set_master(pdev);
3136 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3140 goto err_alloc_etherdev;
3143 SET_NETDEV_DEV(netdev, &pdev->dev);
3145 pci_set_drvdata(pdev, netdev);
3146 adapter = netdev_priv(netdev);
3148 adapter->netdev = netdev;
3149 adapter->pdev = pdev;
3152 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3155 * call save state here in standalone driver because it relies on
3156 * adapter struct to exist, and needs to call netdev_priv
3158 pci_save_state(pdev);
3160 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3161 pci_resource_len(pdev, 0));
3167 ixgbevf_assign_netdev_ops(netdev);
3169 adapter->bd_number = cards_found;
3172 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3173 hw->mac.type = ii->mac;
3175 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3176 sizeof(struct ixgbe_mbx_operations));
3178 /* setup the private structure */
3179 err = ixgbevf_sw_init(adapter);
3183 /* The HW MAC address was set and/or determined in sw_init */
3184 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3186 if (!is_valid_ether_addr(netdev->dev_addr)) {
3187 pr_err("invalid MAC address\n");
3192 netdev->hw_features = NETIF_F_SG |
3199 netdev->features = netdev->hw_features |
3200 NETIF_F_HW_VLAN_TX |
3201 NETIF_F_HW_VLAN_RX |
3202 NETIF_F_HW_VLAN_FILTER;
3204 netdev->vlan_features |= NETIF_F_TSO;
3205 netdev->vlan_features |= NETIF_F_TSO6;
3206 netdev->vlan_features |= NETIF_F_IP_CSUM;
3207 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3208 netdev->vlan_features |= NETIF_F_SG;
3211 netdev->features |= NETIF_F_HIGHDMA;
3213 netdev->priv_flags |= IFF_UNICAST_FLT;
3215 init_timer(&adapter->watchdog_timer);
3216 adapter->watchdog_timer.function = ixgbevf_watchdog;
3217 adapter->watchdog_timer.data = (unsigned long)adapter;
3219 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3220 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3222 err = ixgbevf_init_interrupt_scheme(adapter);
3226 /* pick up the PCI bus settings for reporting later */
3227 if (hw->mac.ops.get_bus_info)
3228 hw->mac.ops.get_bus_info(hw);
3230 strcpy(netdev->name, "eth%d");
3232 err = register_netdev(netdev);
3236 netif_carrier_off(netdev);
3238 ixgbevf_init_last_counter_stats(adapter);
3240 /* print the MAC address */
3241 hw_dbg(hw, "%pM\n", netdev->dev_addr);
3243 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3245 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3250 ixgbevf_clear_interrupt_scheme(adapter);
3252 ixgbevf_reset_interrupt_capability(adapter);
3253 iounmap(hw->hw_addr);
3255 free_netdev(netdev);
3257 pci_release_regions(pdev);
3260 pci_disable_device(pdev);
3265 * ixgbevf_remove - Device Removal Routine
3266 * @pdev: PCI device information struct
3268 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3269 * that it should release a PCI device. The could be caused by a
3270 * Hot-Plug event, or because the driver is going to be removed from
3273 static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3275 struct net_device *netdev = pci_get_drvdata(pdev);
3276 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3278 set_bit(__IXGBEVF_DOWN, &adapter->state);
3280 del_timer_sync(&adapter->watchdog_timer);
3282 cancel_work_sync(&adapter->reset_task);
3283 cancel_work_sync(&adapter->watchdog_task);
3285 if (netdev->reg_state == NETREG_REGISTERED)
3286 unregister_netdev(netdev);
3288 ixgbevf_clear_interrupt_scheme(adapter);
3289 ixgbevf_reset_interrupt_capability(adapter);
3291 iounmap(adapter->hw.hw_addr);
3292 pci_release_regions(pdev);
3294 hw_dbg(&adapter->hw, "Remove complete\n");
3296 kfree(adapter->tx_ring);
3297 kfree(adapter->rx_ring);
3299 free_netdev(netdev);
3301 pci_disable_device(pdev);
3305 * ixgbevf_io_error_detected - called when PCI error is detected
3306 * @pdev: Pointer to PCI device
3307 * @state: The current pci connection state
3309 * This function is called after a PCI bus error affecting
3310 * this device has been detected.
3312 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3313 pci_channel_state_t state)
3315 struct net_device *netdev = pci_get_drvdata(pdev);
3316 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3318 netif_device_detach(netdev);
3320 if (state == pci_channel_io_perm_failure)
3321 return PCI_ERS_RESULT_DISCONNECT;
3323 if (netif_running(netdev))
3324 ixgbevf_down(adapter);
3326 pci_disable_device(pdev);
3328 /* Request a slot slot reset. */
3329 return PCI_ERS_RESULT_NEED_RESET;
3333 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3334 * @pdev: Pointer to PCI device
3336 * Restart the card from scratch, as if from a cold-boot. Implementation
3337 * resembles the first-half of the ixgbevf_resume routine.
3339 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3341 struct net_device *netdev = pci_get_drvdata(pdev);
3342 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3344 if (pci_enable_device_mem(pdev)) {
3346 "Cannot re-enable PCI device after reset.\n");
3347 return PCI_ERS_RESULT_DISCONNECT;
3350 pci_set_master(pdev);
3352 ixgbevf_reset(adapter);
3354 return PCI_ERS_RESULT_RECOVERED;
3358 * ixgbevf_io_resume - called when traffic can start flowing again.
3359 * @pdev: Pointer to PCI device
3361 * This callback is called when the error recovery driver tells us that
3362 * its OK to resume normal operation. Implementation resembles the
3363 * second-half of the ixgbevf_resume routine.
3365 static void ixgbevf_io_resume(struct pci_dev *pdev)
3367 struct net_device *netdev = pci_get_drvdata(pdev);
3368 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3370 if (netif_running(netdev))
3371 ixgbevf_up(adapter);
3373 netif_device_attach(netdev);
3376 /* PCI Error Recovery (ERS) */
3377 static const struct pci_error_handlers ixgbevf_err_handler = {
3378 .error_detected = ixgbevf_io_error_detected,
3379 .slot_reset = ixgbevf_io_slot_reset,
3380 .resume = ixgbevf_io_resume,
3383 static struct pci_driver ixgbevf_driver = {
3384 .name = ixgbevf_driver_name,
3385 .id_table = ixgbevf_pci_tbl,
3386 .probe = ixgbevf_probe,
3387 .remove = __devexit_p(ixgbevf_remove),
3389 /* Power Management Hooks */
3390 .suspend = ixgbevf_suspend,
3391 .resume = ixgbevf_resume,
3393 .shutdown = ixgbevf_shutdown,
3394 .err_handler = &ixgbevf_err_handler
3398 * ixgbevf_init_module - Driver Registration Routine
3400 * ixgbevf_init_module is the first routine called when the driver is
3401 * loaded. All it does is register with the PCI subsystem.
3403 static int __init ixgbevf_init_module(void)
3406 pr_info("%s - version %s\n", ixgbevf_driver_string,
3407 ixgbevf_driver_version);
3409 pr_info("%s\n", ixgbevf_copyright);
3411 ret = pci_register_driver(&ixgbevf_driver);
3415 module_init(ixgbevf_init_module);
3418 * ixgbevf_exit_module - Driver Exit Cleanup Routine
3420 * ixgbevf_exit_module is called just before the driver is removed
3423 static void __exit ixgbevf_exit_module(void)
3425 pci_unregister_driver(&ixgbevf_driver);
3430 * ixgbevf_get_hw_dev_name - return device name string
3431 * used by hardware layer to print debugging information
3433 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3435 struct ixgbevf_adapter *adapter = hw->back;
3436 return adapter->netdev->name;
3440 module_exit(ixgbevf_exit_module);
3442 /* ixgbevf_main.c */