2 * Linux driver for VMware's vmxnet3 ethernet NIC.
4 * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
23 * Maintained by: pv-drivers@vmware.com
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
30 #include "vmxnet3_int.h"
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
37 * Last entry must be all 0s
39 static const struct pci_device_id vmxnet3_pciid_table[] = {
40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
46 static int enable_mq = 1;
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac);
52 * Enable/Disable the given intr
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
69 * Enable/Disable all intrs used by the device
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
78 if (!VMXNET3_VERSION_GE_6(adapter) ||
79 !adapter->queuesExtEnabled) {
80 adapter->shared->devRead.intrConf.intrCtrl &=
81 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
83 adapter->shared->devReadExt.intrConfExt.intrCtrl &=
84 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
90 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
94 if (!VMXNET3_VERSION_GE_6(adapter) ||
95 !adapter->queuesExtEnabled) {
96 adapter->shared->devRead.intrConf.intrCtrl |=
97 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
99 adapter->shared->devReadExt.intrConfExt.intrCtrl |=
100 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
102 for (i = 0; i < adapter->intr.num_intrs; i++)
103 vmxnet3_disable_intr(adapter, i);
108 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
110 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
115 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
122 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
125 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
130 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
133 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
138 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
142 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
145 /* Check if capability is supported by UPT device or
146 * UPT is even requested
149 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap)
151 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) ||
152 cap_supported & (1UL << cap)) {
161 * Check the link state. This may start or stop the tx queue.
164 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
170 spin_lock_irqsave(&adapter->cmd_lock, flags);
171 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
172 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
173 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
175 adapter->link_speed = ret >> 16;
176 if (ret & 1) { /* Link is up. */
177 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
178 adapter->link_speed);
179 netif_carrier_on(adapter->netdev);
182 for (i = 0; i < adapter->num_tx_queues; i++)
183 vmxnet3_tq_start(&adapter->tx_queue[i],
187 netdev_info(adapter->netdev, "NIC Link is Down\n");
188 netif_carrier_off(adapter->netdev);
191 for (i = 0; i < adapter->num_tx_queues; i++)
192 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
198 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
202 u32 events = le32_to_cpu(adapter->shared->ecr);
206 vmxnet3_ack_events(adapter, events);
208 /* Check if link state has changed */
209 if (events & VMXNET3_ECR_LINK)
210 vmxnet3_check_link(adapter, true);
212 /* Check if there is an error on xmit/recv queues */
213 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
214 spin_lock_irqsave(&adapter->cmd_lock, flags);
215 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
216 VMXNET3_CMD_GET_QUEUE_STATUS);
217 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
219 for (i = 0; i < adapter->num_tx_queues; i++)
220 if (adapter->tqd_start[i].status.stopped)
221 dev_err(&adapter->netdev->dev,
222 "%s: tq[%d] error 0x%x\n",
223 adapter->netdev->name, i, le32_to_cpu(
224 adapter->tqd_start[i].status.error));
225 for (i = 0; i < adapter->num_rx_queues; i++)
226 if (adapter->rqd_start[i].status.stopped)
227 dev_err(&adapter->netdev->dev,
228 "%s: rq[%d] error 0x%x\n",
229 adapter->netdev->name, i,
230 adapter->rqd_start[i].status.error);
232 schedule_work(&adapter->work);
236 #ifdef __BIG_ENDIAN_BITFIELD
238 * The device expects the bitfields in shared structures to be written in
239 * little endian. When CPU is big endian, the following routines are used to
240 * correctly read and write into ABI.
241 * The general technique used here is : double word bitfields are defined in
242 * opposite order for big endian architecture. Then before reading them in
243 * driver the complete double word is translated using le32_to_cpu. Similarly
244 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
245 * double words into required format.
246 * In order to avoid touching bits in shared structure more than once, temporary
247 * descriptors are used. These are passed as srcDesc to following functions.
249 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
250 struct Vmxnet3_RxDesc *dstDesc)
252 u32 *src = (u32 *)srcDesc + 2;
253 u32 *dst = (u32 *)dstDesc + 2;
254 dstDesc->addr = le64_to_cpu(srcDesc->addr);
255 *dst = le32_to_cpu(*src);
256 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
259 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
260 struct Vmxnet3_TxDesc *dstDesc)
263 u32 *src = (u32 *)(srcDesc + 1);
264 u32 *dst = (u32 *)(dstDesc + 1);
266 /* Working backwards so that the gen bit is set at the end. */
267 for (i = 2; i > 0; i--) {
270 *dst = cpu_to_le32(*src);
275 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
276 struct Vmxnet3_RxCompDesc *dstDesc)
279 u32 *src = (u32 *)srcDesc;
280 u32 *dst = (u32 *)dstDesc;
281 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
282 *dst = le32_to_cpu(*src);
289 /* Used to read bitfield values from double words. */
290 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
292 u32 temp = le32_to_cpu(*bitfield);
293 u32 mask = ((1 << size) - 1) << pos;
301 #endif /* __BIG_ENDIAN_BITFIELD */
303 #ifdef __BIG_ENDIAN_BITFIELD
305 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
306 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
307 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
308 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
309 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
310 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
311 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
312 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
313 VMXNET3_TCD_GEN_SIZE)
314 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
315 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
316 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
318 vmxnet3_RxCompToCPU((rcd), (tmp)); \
320 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
322 vmxnet3_RxDescToCPU((rxd), (tmp)); \
327 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
328 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
329 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
330 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
331 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
332 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
334 #endif /* __BIG_ENDIAN_BITFIELD */
338 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
339 struct pci_dev *pdev)
341 if (tbi->map_type == VMXNET3_MAP_SINGLE)
342 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
344 else if (tbi->map_type == VMXNET3_MAP_PAGE)
345 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
348 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
350 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
355 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
356 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
361 /* no out of order completion */
362 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
363 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
365 skb = tq->buf_info[eop_idx].skb;
367 tq->buf_info[eop_idx].skb = NULL;
369 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
371 while (tq->tx_ring.next2comp != eop_idx) {
372 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
375 /* update next2comp w/o tx_lock. Since we are marking more,
376 * instead of less, tx ring entries avail, the worst case is
377 * that the tx routine incorrectly re-queues a pkt due to
378 * insufficient tx ring entries.
380 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
384 dev_kfree_skb_any(skb);
390 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
391 struct vmxnet3_adapter *adapter)
394 union Vmxnet3_GenericDesc *gdesc;
396 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
397 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
398 /* Prevent any &gdesc->tcd field from being (speculatively)
399 * read before (&gdesc->tcd)->gen is read.
403 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
404 &gdesc->tcd), tq, adapter->pdev,
407 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
408 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
412 spin_lock(&tq->tx_lock);
413 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
414 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
415 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
416 netif_carrier_ok(adapter->netdev))) {
417 vmxnet3_tq_wake(tq, adapter);
419 spin_unlock(&tq->tx_lock);
426 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
427 struct vmxnet3_adapter *adapter)
431 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
432 struct vmxnet3_tx_buf_info *tbi;
434 tbi = tq->buf_info + tq->tx_ring.next2comp;
436 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
438 dev_kfree_skb_any(tbi->skb);
441 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
444 /* sanity check, verify all buffers are indeed unmapped and freed */
445 for (i = 0; i < tq->tx_ring.size; i++) {
446 BUG_ON(tq->buf_info[i].skb != NULL ||
447 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
450 tq->tx_ring.gen = VMXNET3_INIT_GEN;
451 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
453 tq->comp_ring.gen = VMXNET3_INIT_GEN;
454 tq->comp_ring.next2proc = 0;
459 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
460 struct vmxnet3_adapter *adapter)
462 if (tq->tx_ring.base) {
463 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
464 sizeof(struct Vmxnet3_TxDesc),
465 tq->tx_ring.base, tq->tx_ring.basePA);
466 tq->tx_ring.base = NULL;
468 if (tq->data_ring.base) {
469 dma_free_coherent(&adapter->pdev->dev,
470 tq->data_ring.size * tq->txdata_desc_size,
471 tq->data_ring.base, tq->data_ring.basePA);
472 tq->data_ring.base = NULL;
474 if (tq->comp_ring.base) {
475 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
476 sizeof(struct Vmxnet3_TxCompDesc),
477 tq->comp_ring.base, tq->comp_ring.basePA);
478 tq->comp_ring.base = NULL;
485 /* Destroy all tx queues */
487 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
491 for (i = 0; i < adapter->num_tx_queues; i++)
492 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
497 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
498 struct vmxnet3_adapter *adapter)
502 /* reset the tx ring contents to 0 and reset the tx ring states */
503 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
504 sizeof(struct Vmxnet3_TxDesc));
505 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
506 tq->tx_ring.gen = VMXNET3_INIT_GEN;
508 memset(tq->data_ring.base, 0,
509 tq->data_ring.size * tq->txdata_desc_size);
511 /* reset the tx comp ring contents to 0 and reset comp ring states */
512 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
513 sizeof(struct Vmxnet3_TxCompDesc));
514 tq->comp_ring.next2proc = 0;
515 tq->comp_ring.gen = VMXNET3_INIT_GEN;
517 /* reset the bookkeeping data */
518 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
519 for (i = 0; i < tq->tx_ring.size; i++)
520 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
522 /* stats are not reset */
527 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
528 struct vmxnet3_adapter *adapter)
530 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
531 tq->comp_ring.base || tq->buf_info);
533 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
534 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
535 &tq->tx_ring.basePA, GFP_KERNEL);
536 if (!tq->tx_ring.base) {
537 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
541 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
542 tq->data_ring.size * tq->txdata_desc_size,
543 &tq->data_ring.basePA, GFP_KERNEL);
544 if (!tq->data_ring.base) {
545 netdev_err(adapter->netdev, "failed to allocate tx data ring\n");
549 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
550 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
551 &tq->comp_ring.basePA, GFP_KERNEL);
552 if (!tq->comp_ring.base) {
553 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
557 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]),
559 dev_to_node(&adapter->pdev->dev));
566 vmxnet3_tq_destroy(tq, adapter);
571 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
575 for (i = 0; i < adapter->num_tx_queues; i++)
576 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
580 * starting from ring->next2fill, allocate rx buffers for the given ring
581 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
582 * are allocated or allocation fails
586 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
587 int num_to_alloc, struct vmxnet3_adapter *adapter)
589 int num_allocated = 0;
590 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
591 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
594 while (num_allocated <= num_to_alloc) {
595 struct vmxnet3_rx_buf_info *rbi;
596 union Vmxnet3_GenericDesc *gd;
598 rbi = rbi_base + ring->next2fill;
599 gd = ring->base + ring->next2fill;
600 rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
602 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
603 if (rbi->skb == NULL) {
604 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
607 if (unlikely(rbi->skb == NULL)) {
608 rq->stats.rx_buf_alloc_failure++;
612 rbi->dma_addr = dma_map_single(
614 rbi->skb->data, rbi->len,
616 if (dma_mapping_error(&adapter->pdev->dev,
618 dev_kfree_skb_any(rbi->skb);
620 rq->stats.rx_buf_alloc_failure++;
624 /* rx buffer skipped by the device */
626 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
628 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
629 rbi->len != PAGE_SIZE);
631 if (rbi->page == NULL) {
632 rbi->page = alloc_page(GFP_ATOMIC);
633 if (unlikely(rbi->page == NULL)) {
634 rq->stats.rx_buf_alloc_failure++;
637 rbi->dma_addr = dma_map_page(
639 rbi->page, 0, PAGE_SIZE,
641 if (dma_mapping_error(&adapter->pdev->dev,
645 rq->stats.rx_buf_alloc_failure++;
649 /* rx buffers skipped by the device */
651 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
654 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
655 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
658 /* Fill the last buffer but dont mark it ready, or else the
659 * device will think that the queue is full */
660 if (num_allocated == num_to_alloc) {
661 rbi->comp_state = VMXNET3_RXD_COMP_DONE;
665 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
667 vmxnet3_cmd_ring_adv_next2fill(ring);
670 netdev_dbg(adapter->netdev,
671 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
672 num_allocated, ring->next2fill, ring->next2comp);
674 /* so that the device can distinguish a full ring and an empty ring */
675 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
677 return num_allocated;
682 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
683 struct vmxnet3_rx_buf_info *rbi)
685 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
687 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
689 __skb_frag_set_page(frag, rbi->page);
690 skb_frag_off_set(frag, 0);
691 skb_frag_size_set(frag, rcd->len);
692 skb->data_len += rcd->len;
693 skb->truesize += PAGE_SIZE;
694 skb_shinfo(skb)->nr_frags++;
699 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
700 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
701 struct vmxnet3_adapter *adapter)
704 unsigned long buf_offset;
706 union Vmxnet3_GenericDesc *gdesc;
707 struct vmxnet3_tx_buf_info *tbi = NULL;
709 BUG_ON(ctx->copy_size > skb_headlen(skb));
711 /* use the previous gen bit for the SOP desc */
712 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
714 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
715 gdesc = ctx->sop_txd; /* both loops below can be skipped */
717 /* no need to map the buffer if headers are copied */
718 if (ctx->copy_size) {
719 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
720 tq->tx_ring.next2fill *
721 tq->txdata_desc_size);
722 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
723 ctx->sop_txd->dword[3] = 0;
725 tbi = tq->buf_info + tq->tx_ring.next2fill;
726 tbi->map_type = VMXNET3_MAP_NONE;
728 netdev_dbg(adapter->netdev,
729 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 tq->tx_ring.next2fill,
731 le64_to_cpu(ctx->sop_txd->txd.addr),
732 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
733 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
735 /* use the right gen for non-SOP desc */
736 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
739 /* linear part can use multiple tx desc if it's big */
740 len = skb_headlen(skb) - ctx->copy_size;
741 buf_offset = ctx->copy_size;
745 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
749 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
750 /* spec says that for TxDesc.len, 0 == 2^14 */
753 tbi = tq->buf_info + tq->tx_ring.next2fill;
754 tbi->map_type = VMXNET3_MAP_SINGLE;
755 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
756 skb->data + buf_offset, buf_size,
758 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
763 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
764 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
766 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
767 gdesc->dword[2] = cpu_to_le32(dw2);
770 netdev_dbg(adapter->netdev,
771 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
772 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
773 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
774 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
775 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
778 buf_offset += buf_size;
781 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
782 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
786 len = skb_frag_size(frag);
788 tbi = tq->buf_info + tq->tx_ring.next2fill;
789 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
793 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
794 /* spec says that for TxDesc.len, 0 == 2^14 */
796 tbi->map_type = VMXNET3_MAP_PAGE;
797 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
798 buf_offset, buf_size,
800 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
805 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
806 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
808 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
809 gdesc->dword[2] = cpu_to_le32(dw2);
812 netdev_dbg(adapter->netdev,
813 "txd[%u]: 0x%llx %u %u\n",
814 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
815 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
816 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
817 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
820 buf_offset += buf_size;
824 ctx->eop_txd = gdesc;
826 /* set the last buf_info for the pkt */
828 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
834 /* Init all tx queues */
836 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
840 for (i = 0; i < adapter->num_tx_queues; i++)
841 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
846 * parse relevant protocol headers:
847 * For a tso pkt, relevant headers are L2/3/4 including options
848 * For a pkt requesting csum offloading, they are L2/3 and may include L4
849 * if it's a TCP/UDP pkt
852 * -1: error happens during parsing
853 * 0: protocol headers parsed, but too big to be copied
854 * 1: protocol headers parsed and copied
857 * 1. related *ctx fields are updated.
858 * 2. ctx->copy_size is # of bytes copied
859 * 3. the portion to be copied is guaranteed to be in the linear part
863 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
864 struct vmxnet3_tx_ctx *ctx,
865 struct vmxnet3_adapter *adapter)
869 if (ctx->mss) { /* TSO */
870 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
871 ctx->l4_offset = skb_inner_transport_offset(skb);
872 ctx->l4_hdr_size = inner_tcp_hdrlen(skb);
873 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
875 ctx->l4_offset = skb_transport_offset(skb);
876 ctx->l4_hdr_size = tcp_hdrlen(skb);
877 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
880 if (skb->ip_summed == CHECKSUM_PARTIAL) {
881 /* For encap packets, skb_checksum_start_offset refers
882 * to inner L4 offset. Thus, below works for encap as
883 * well as non-encap case
885 ctx->l4_offset = skb_checksum_start_offset(skb);
887 if (VMXNET3_VERSION_GE_4(adapter) &&
888 skb->encapsulation) {
889 struct iphdr *iph = inner_ip_hdr(skb);
891 if (iph->version == 4) {
892 protocol = iph->protocol;
894 const struct ipv6hdr *ipv6h;
896 ipv6h = inner_ipv6_hdr(skb);
897 protocol = ipv6h->nexthdr;
901 const struct iphdr *iph = ip_hdr(skb);
903 protocol = iph->protocol;
904 } else if (ctx->ipv6) {
905 const struct ipv6hdr *ipv6h;
907 ipv6h = ipv6_hdr(skb);
908 protocol = ipv6h->nexthdr;
914 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) :
918 ctx->l4_hdr_size = sizeof(struct udphdr);
921 ctx->l4_hdr_size = 0;
925 ctx->copy_size = min(ctx->l4_offset +
926 ctx->l4_hdr_size, skb->len);
929 ctx->l4_hdr_size = 0;
930 /* copy as much as allowed */
931 ctx->copy_size = min_t(unsigned int,
932 tq->txdata_desc_size,
936 if (skb->len <= VMXNET3_HDR_COPY_SIZE)
937 ctx->copy_size = skb->len;
939 /* make sure headers are accessible directly */
940 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
944 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {
945 tq->stats.oversized_hdr++;
956 * copy relevant protocol headers to the transmit ring:
957 * For a tso pkt, relevant headers are L2/3/4 including options
958 * For a pkt requesting csum offloading, they are L2/3 and may include L4
959 * if it's a TCP/UDP pkt
962 * Note that this requires that vmxnet3_parse_hdr be called first to set the
963 * appropriate bits in ctx first
966 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
967 struct vmxnet3_tx_ctx *ctx,
968 struct vmxnet3_adapter *adapter)
970 struct Vmxnet3_TxDataDesc *tdd;
972 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +
973 tq->tx_ring.next2fill *
974 tq->txdata_desc_size);
976 memcpy(tdd->data, skb->data, ctx->copy_size);
977 netdev_dbg(adapter->netdev,
978 "copy %u bytes to dataRing[%u]\n",
979 ctx->copy_size, tq->tx_ring.next2fill);
984 vmxnet3_prepare_inner_tso(struct sk_buff *skb,
985 struct vmxnet3_tx_ctx *ctx)
987 struct tcphdr *tcph = inner_tcp_hdr(skb);
988 struct iphdr *iph = inner_ip_hdr(skb);
990 if (iph->version == 4) {
992 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
995 struct ipv6hdr *iph = inner_ipv6_hdr(skb);
997 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
1003 vmxnet3_prepare_tso(struct sk_buff *skb,
1004 struct vmxnet3_tx_ctx *ctx)
1006 struct tcphdr *tcph = tcp_hdr(skb);
1009 struct iphdr *iph = ip_hdr(skb);
1012 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
1014 } else if (ctx->ipv6) {
1015 tcp_v6_gso_csum_prep(skb);
1019 static int txd_estimate(const struct sk_buff *skb)
1021 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1025 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1027 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
1033 * Transmits a pkt thru a given tq
1035 * NETDEV_TX_OK: descriptors are setup successfully
1036 * NETDEV_TX_OK: error occurred, the pkt is dropped
1037 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
1040 * 1. tx ring may be changed
1041 * 2. tq stats may be updated accordingly
1042 * 3. shared->txNumDeferred may be updated
1046 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1047 struct vmxnet3_adapter *adapter, struct net_device *netdev)
1052 int tx_num_deferred;
1053 unsigned long flags;
1054 struct vmxnet3_tx_ctx ctx;
1055 union Vmxnet3_GenericDesc *gdesc;
1056 #ifdef __BIG_ENDIAN_BITFIELD
1057 /* Use temporary descriptor to avoid touching bits multiple times */
1058 union Vmxnet3_GenericDesc tempTxDesc;
1061 count = txd_estimate(skb);
1063 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
1064 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6));
1066 ctx.mss = skb_shinfo(skb)->gso_size;
1068 if (skb_header_cloned(skb)) {
1069 if (unlikely(pskb_expand_head(skb, 0, 0,
1070 GFP_ATOMIC) != 0)) {
1071 tq->stats.drop_tso++;
1074 tq->stats.copy_skb_header++;
1076 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1077 /* tso pkts must not use more than
1078 * VMXNET3_MAX_TSO_TXD_PER_PKT entries
1080 if (skb_linearize(skb) != 0) {
1081 tq->stats.drop_too_many_frags++;
1084 tq->stats.linearized++;
1086 /* recalculate the # of descriptors to use */
1087 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1088 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1089 tq->stats.drop_too_many_frags++;
1093 if (skb->encapsulation) {
1094 vmxnet3_prepare_inner_tso(skb, &ctx);
1096 vmxnet3_prepare_tso(skb, &ctx);
1099 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
1101 /* non-tso pkts must not use more than
1102 * VMXNET3_MAX_TXD_PER_PKT entries
1104 if (skb_linearize(skb) != 0) {
1105 tq->stats.drop_too_many_frags++;
1108 tq->stats.linearized++;
1110 /* recalculate the # of descriptors to use */
1111 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1115 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter);
1117 BUG_ON(ret <= 0 && ctx.copy_size != 0);
1118 /* hdrs parsed, check against other limits */
1120 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size >
1121 VMXNET3_MAX_TX_BUF_SIZE)) {
1122 tq->stats.drop_oversized_hdr++;
1126 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1127 if (unlikely(ctx.l4_offset +
1129 VMXNET3_MAX_CSUM_OFFSET)) {
1130 tq->stats.drop_oversized_hdr++;
1136 tq->stats.drop_hdr_inspect_err++;
1140 spin_lock_irqsave(&tq->tx_lock, flags);
1142 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
1143 tq->stats.tx_ring_full++;
1144 netdev_dbg(adapter->netdev,
1145 "tx queue stopped on %s, next2comp %u"
1146 " next2fill %u\n", adapter->netdev->name,
1147 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
1149 vmxnet3_tq_stop(tq, adapter);
1150 spin_unlock_irqrestore(&tq->tx_lock, flags);
1151 return NETDEV_TX_BUSY;
1155 vmxnet3_copy_hdr(skb, tq, &ctx, adapter);
1157 /* fill tx descs related to addr & len */
1158 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter))
1159 goto unlock_drop_pkt;
1161 /* setup the EOP desc */
1162 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1164 /* setup the SOP desc */
1165 #ifdef __BIG_ENDIAN_BITFIELD
1166 gdesc = &tempTxDesc;
1167 gdesc->dword[2] = ctx.sop_txd->dword[2];
1168 gdesc->dword[3] = ctx.sop_txd->dword[3];
1170 gdesc = ctx.sop_txd;
1172 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
1174 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
1175 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1176 if (VMXNET3_VERSION_GE_7(adapter)) {
1177 gdesc->txd.om = VMXNET3_OM_TSO;
1178 gdesc->txd.ext1 = 1;
1180 gdesc->txd.om = VMXNET3_OM_ENCAP;
1182 gdesc->txd.msscof = ctx.mss;
1184 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1187 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1188 gdesc->txd.om = VMXNET3_OM_TSO;
1189 gdesc->txd.msscof = ctx.mss;
1191 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
1193 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1194 if (VMXNET3_VERSION_GE_4(adapter) &&
1195 skb->encapsulation) {
1196 gdesc->txd.hlen = ctx.l4_offset +
1198 if (VMXNET3_VERSION_GE_7(adapter)) {
1199 gdesc->txd.om = VMXNET3_OM_CSUM;
1200 gdesc->txd.msscof = ctx.l4_offset +
1202 gdesc->txd.ext1 = 1;
1204 gdesc->txd.om = VMXNET3_OM_ENCAP;
1205 gdesc->txd.msscof = 0; /* Reserved */
1208 gdesc->txd.hlen = ctx.l4_offset;
1209 gdesc->txd.om = VMXNET3_OM_CSUM;
1210 gdesc->txd.msscof = ctx.l4_offset +
1215 gdesc->txd.msscof = 0;
1219 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
1220 tx_num_deferred += num_pkts;
1222 if (skb_vlan_tag_present(skb)) {
1224 gdesc->txd.tci = skb_vlan_tag_get(skb);
1227 /* Ensure that the write to (&gdesc->txd)->gen will be observed after
1228 * all other writes to &gdesc->txd.
1232 /* finally flips the GEN bit of the SOP desc. */
1233 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1235 #ifdef __BIG_ENDIAN_BITFIELD
1236 /* Finished updating in bitfields of Tx Desc, so write them in original
1239 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1240 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1241 gdesc = ctx.sop_txd;
1243 netdev_dbg(adapter->netdev,
1244 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1246 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1247 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1249 spin_unlock_irqrestore(&tq->tx_lock, flags);
1251 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
1252 tq->shared->txNumDeferred = 0;
1253 VMXNET3_WRITE_BAR0_REG(adapter,
1254 adapter->tx_prod_offset + tq->qid * 8,
1255 tq->tx_ring.next2fill);
1258 return NETDEV_TX_OK;
1261 spin_unlock_irqrestore(&tq->tx_lock, flags);
1263 tq->stats.drop_total++;
1264 dev_kfree_skb_any(skb);
1265 return NETDEV_TX_OK;
1270 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1272 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1274 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1275 return vmxnet3_tq_xmit(skb,
1276 &adapter->tx_queue[skb->queue_mapping],
1282 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1283 struct sk_buff *skb,
1284 union Vmxnet3_GenericDesc *gdesc)
1286 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1287 if (gdesc->rcd.v4 &&
1288 (le32_to_cpu(gdesc->dword[3]) &
1289 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) {
1290 skb->ip_summed = CHECKSUM_UNNECESSARY;
1291 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1292 !(le32_to_cpu(gdesc->dword[0]) &
1293 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1294 WARN_ON_ONCE(gdesc->rcd.frg &&
1295 !(le32_to_cpu(gdesc->dword[0]) &
1296 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1297 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) &
1298 (1 << VMXNET3_RCD_TUC_SHIFT))) {
1299 skb->ip_summed = CHECKSUM_UNNECESSARY;
1300 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1301 !(le32_to_cpu(gdesc->dword[0]) &
1302 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1303 WARN_ON_ONCE(gdesc->rcd.frg &&
1304 !(le32_to_cpu(gdesc->dword[0]) &
1305 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1307 if (gdesc->rcd.csum) {
1308 skb->csum = htons(gdesc->rcd.csum);
1309 skb->ip_summed = CHECKSUM_PARTIAL;
1311 skb_checksum_none_assert(skb);
1315 skb_checksum_none_assert(skb);
1321 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1322 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1324 rq->stats.drop_err++;
1326 rq->stats.drop_fcs++;
1328 rq->stats.drop_total++;
1331 * We do not unmap and chain the rx buffer to the skb.
1332 * We basically pretend this buffer is not used and will be recycled
1333 * by vmxnet3_rq_alloc_rx_buf()
1337 * ctx->skb may be NULL if this is the first and the only one
1341 dev_kfree_skb_irq(ctx->skb);
1348 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
1349 union Vmxnet3_GenericDesc *gdesc)
1355 struct vlan_ethhdr *veth;
1357 struct ipv6hdr *ipv6;
1360 BUG_ON(gdesc->rcd.tcp == 0);
1362 maplen = skb_headlen(skb);
1363 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
1366 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
1367 skb->protocol == cpu_to_be16(ETH_P_8021AD))
1368 hlen = sizeof(struct vlan_ethhdr);
1370 hlen = sizeof(struct ethhdr);
1372 hdr.eth = eth_hdr(skb);
1373 if (gdesc->rcd.v4) {
1374 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
1375 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
1377 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
1378 hlen = hdr.ipv4->ihl << 2;
1379 hdr.ptr += hdr.ipv4->ihl << 2;
1380 } else if (gdesc->rcd.v6) {
1381 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
1382 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
1384 /* Use an estimated value, since we also need to handle
1387 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1388 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
1389 hlen = sizeof(struct ipv6hdr);
1390 hdr.ptr += sizeof(struct ipv6hdr);
1392 /* Non-IP pkt, dont estimate header length */
1396 if (hlen + sizeof(struct tcphdr) > maplen)
1399 return (hlen + (hdr.tcp->doff << 2));
1403 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1404 struct vmxnet3_adapter *adapter, int quota)
1406 u32 rxprod_reg[2] = {
1407 adapter->rx_prod_offset, adapter->rx_prod2_offset
1410 bool skip_page_frags = false;
1411 bool encap_lro = false;
1412 struct Vmxnet3_RxCompDesc *rcd;
1413 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1414 u16 segCnt = 0, mss = 0;
1415 int comp_offset, fill_offset;
1416 #ifdef __BIG_ENDIAN_BITFIELD
1417 struct Vmxnet3_RxDesc rxCmdDesc;
1418 struct Vmxnet3_RxCompDesc rxComp;
1420 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1422 while (rcd->gen == rq->comp_ring.gen) {
1423 struct vmxnet3_rx_buf_info *rbi;
1424 struct sk_buff *skb, *new_skb = NULL;
1425 struct page *new_page = NULL;
1426 dma_addr_t new_dma_addr;
1428 struct Vmxnet3_RxDesc *rxd;
1430 struct vmxnet3_cmd_ring *ring = NULL;
1431 if (num_pkts >= quota) {
1432 /* we may stop even before we see the EOP desc of
1438 /* Prevent any rcd field from being (speculatively) read before
1443 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 &&
1444 rcd->rqID != rq->dataRingQid);
1446 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID);
1447 ring = rq->rx_ring + ring_idx;
1448 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1450 rbi = rq->buf_info[ring_idx] + idx;
1452 BUG_ON(rxd->addr != rbi->dma_addr ||
1453 rxd->len != rbi->len);
1455 if (unlikely(rcd->eop && rcd->err)) {
1456 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1460 if (rcd->sop) { /* first buf of the pkt */
1461 bool rxDataRingUsed;
1464 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1465 (rcd->rqID != rq->qid &&
1466 rcd->rqID != rq->dataRingQid));
1468 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1469 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1471 if (unlikely(rcd->len == 0)) {
1472 /* Pretend the rx buffer is skipped. */
1473 BUG_ON(!(rcd->sop && rcd->eop));
1474 netdev_dbg(adapter->netdev,
1475 "rxRing[%u][%u] 0 length\n",
1480 skip_page_frags = false;
1481 ctx->skb = rbi->skb;
1484 VMXNET3_RX_DATA_RING(adapter, rcd->rqID);
1485 len = rxDataRingUsed ? rcd->len : rbi->len;
1486 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1488 if (new_skb == NULL) {
1489 /* Skb allocation failed, do not handover this
1490 * skb to stack. Reuse it. Drop the existing pkt
1492 rq->stats.rx_buf_alloc_failure++;
1494 rq->stats.drop_total++;
1495 skip_page_frags = true;
1499 if (rxDataRingUsed) {
1502 BUG_ON(rcd->len > rq->data_ring.desc_size);
1505 sz = rcd->rxdIdx * rq->data_ring.desc_size;
1506 memcpy(new_skb->data,
1507 &rq->data_ring.base[sz], rcd->len);
1509 ctx->skb = rbi->skb;
1512 dma_map_single(&adapter->pdev->dev,
1513 new_skb->data, rbi->len,
1515 if (dma_mapping_error(&adapter->pdev->dev,
1517 dev_kfree_skb(new_skb);
1518 /* Skb allocation failed, do not
1519 * handover this skb to stack. Reuse
1520 * it. Drop the existing pkt.
1522 rq->stats.rx_buf_alloc_failure++;
1524 rq->stats.drop_total++;
1525 skip_page_frags = true;
1529 dma_unmap_single(&adapter->pdev->dev,
1534 /* Immediate refill */
1536 rbi->dma_addr = new_dma_addr;
1537 rxd->addr = cpu_to_le64(rbi->dma_addr);
1538 rxd->len = rbi->len;
1542 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1543 (adapter->netdev->features & NETIF_F_RXHASH)) {
1544 enum pkt_hash_types hash_type;
1546 switch (rcd->rssType) {
1547 case VMXNET3_RCD_RSS_TYPE_IPV4:
1548 case VMXNET3_RCD_RSS_TYPE_IPV6:
1549 hash_type = PKT_HASH_TYPE_L3;
1551 case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1552 case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1553 case VMXNET3_RCD_RSS_TYPE_UDPIPV4:
1554 case VMXNET3_RCD_RSS_TYPE_UDPIPV6:
1555 hash_type = PKT_HASH_TYPE_L4;
1558 hash_type = PKT_HASH_TYPE_L3;
1561 skb_set_hash(ctx->skb,
1562 le32_to_cpu(rcd->rssHash),
1566 skb_record_rx_queue(ctx->skb, rq->qid);
1567 skb_put(ctx->skb, rcd->len);
1569 if (VMXNET3_VERSION_GE_2(adapter) &&
1570 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
1571 struct Vmxnet3_RxCompDescExt *rcdlro;
1572 union Vmxnet3_GenericDesc *gdesc;
1574 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd;
1575 gdesc = (union Vmxnet3_GenericDesc *)rcd;
1577 segCnt = rcdlro->segCnt;
1578 WARN_ON_ONCE(segCnt == 0);
1580 if (unlikely(segCnt <= 1))
1582 encap_lro = (le32_to_cpu(gdesc->dword[0]) &
1583 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT));
1588 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1590 /* non SOP buffer must be type 1 in most cases */
1591 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1592 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1594 /* If an sop buffer was dropped, skip all
1595 * following non-sop fragments. They will be reused.
1597 if (skip_page_frags)
1601 new_page = alloc_page(GFP_ATOMIC);
1602 /* Replacement page frag could not be allocated.
1603 * Reuse this page. Drop the pkt and free the
1604 * skb which contained this page as a frag. Skip
1605 * processing all the following non-sop frags.
1607 if (unlikely(!new_page)) {
1608 rq->stats.rx_buf_alloc_failure++;
1609 dev_kfree_skb(ctx->skb);
1611 skip_page_frags = true;
1614 new_dma_addr = dma_map_page(&adapter->pdev->dev,
1618 if (dma_mapping_error(&adapter->pdev->dev,
1621 rq->stats.rx_buf_alloc_failure++;
1622 dev_kfree_skb(ctx->skb);
1624 skip_page_frags = true;
1628 dma_unmap_page(&adapter->pdev->dev,
1629 rbi->dma_addr, rbi->len,
1632 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1634 /* Immediate refill */
1635 rbi->page = new_page;
1636 rbi->dma_addr = new_dma_addr;
1637 rxd->addr = cpu_to_le64(rbi->dma_addr);
1638 rxd->len = rbi->len;
1645 u32 mtu = adapter->netdev->mtu;
1646 skb->len += skb->data_len;
1648 vmxnet3_rx_csum(adapter, skb,
1649 (union Vmxnet3_GenericDesc *)rcd);
1650 skb->protocol = eth_type_trans(skb, adapter->netdev);
1651 if ((!rcd->tcp && !encap_lro) ||
1652 !(adapter->netdev->features & NETIF_F_LRO))
1655 if (segCnt != 0 && mss != 0) {
1656 skb_shinfo(skb)->gso_type = rcd->v4 ?
1657 SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1658 skb_shinfo(skb)->gso_size = mss;
1659 skb_shinfo(skb)->gso_segs = segCnt;
1660 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) {
1663 hlen = vmxnet3_get_hdr_len(adapter, skb,
1664 (union Vmxnet3_GenericDesc *)rcd);
1668 skb_shinfo(skb)->gso_type =
1669 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1671 skb_shinfo(skb)->gso_segs = segCnt;
1672 skb_shinfo(skb)->gso_size =
1673 DIV_ROUND_UP(skb->len -
1676 skb_shinfo(skb)->gso_size = mtu - hlen;
1680 if (unlikely(rcd->ts))
1681 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1683 if (adapter->netdev->features & NETIF_F_LRO)
1684 netif_receive_skb(skb);
1686 napi_gro_receive(&rq->napi, skb);
1694 /* device may have skipped some rx descs */
1695 ring = rq->rx_ring + ring_idx;
1696 rbi->comp_state = VMXNET3_RXD_COMP_DONE;
1698 comp_offset = vmxnet3_cmd_ring_desc_avail(ring);
1699 fill_offset = (idx > ring->next2fill ? 0 : ring->size) +
1700 idx - ring->next2fill - 1;
1701 if (!ring->isOutOfOrder || fill_offset >= comp_offset)
1702 ring->next2comp = idx;
1703 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1705 /* Ensure that the writes to rxd->gen bits will be observed
1706 * after all other writes to rxd objects.
1710 while (num_to_alloc) {
1711 rbi = rq->buf_info[ring_idx] + ring->next2fill;
1712 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP)))
1714 if (ring_idx == 0) {
1715 /* ring0 Type1 buffers can get skipped; re-fill them */
1716 if (rbi->buf_type != VMXNET3_RX_BUF_SKB)
1719 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) {
1721 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1723 WARN_ON(!rxd->addr);
1725 /* Recv desc is ready to be used by the device */
1726 rxd->gen = ring->gen;
1727 vmxnet3_cmd_ring_adv_next2fill(ring);
1728 rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
1731 /* rx completion hasn't occurred */
1732 ring->isOutOfOrder = 1;
1737 if (num_to_alloc == 0) {
1738 ring->isOutOfOrder = 0;
1741 /* if needed, update the register */
1742 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) {
1743 VMXNET3_WRITE_BAR0_REG(adapter,
1744 rxprod_reg[ring_idx] + rq->qid * 8,
1748 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1749 vmxnet3_getRxComp(rcd,
1750 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1758 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1759 struct vmxnet3_adapter *adapter)
1762 struct Vmxnet3_RxDesc *rxd;
1764 /* ring has already been cleaned up */
1765 if (!rq->rx_ring[0].base)
1768 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1769 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1770 #ifdef __BIG_ENDIAN_BITFIELD
1771 struct Vmxnet3_RxDesc rxDesc;
1773 vmxnet3_getRxDesc(rxd,
1774 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1776 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1777 rq->buf_info[ring_idx][i].skb) {
1778 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1779 rxd->len, DMA_FROM_DEVICE);
1780 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1781 rq->buf_info[ring_idx][i].skb = NULL;
1782 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1783 rq->buf_info[ring_idx][i].page) {
1784 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1785 rxd->len, DMA_FROM_DEVICE);
1786 put_page(rq->buf_info[ring_idx][i].page);
1787 rq->buf_info[ring_idx][i].page = NULL;
1791 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1792 rq->rx_ring[ring_idx].next2fill =
1793 rq->rx_ring[ring_idx].next2comp = 0;
1796 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1797 rq->comp_ring.next2proc = 0;
1802 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1806 for (i = 0; i < adapter->num_rx_queues; i++)
1807 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1811 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1812 struct vmxnet3_adapter *adapter)
1817 /* all rx buffers must have already been freed */
1818 for (i = 0; i < 2; i++) {
1819 if (rq->buf_info[i]) {
1820 for (j = 0; j < rq->rx_ring[i].size; j++)
1821 BUG_ON(rq->buf_info[i][j].page != NULL);
1826 for (i = 0; i < 2; i++) {
1827 if (rq->rx_ring[i].base) {
1828 dma_free_coherent(&adapter->pdev->dev,
1830 * sizeof(struct Vmxnet3_RxDesc),
1831 rq->rx_ring[i].base,
1832 rq->rx_ring[i].basePA);
1833 rq->rx_ring[i].base = NULL;
1837 if (rq->data_ring.base) {
1838 dma_free_coherent(&adapter->pdev->dev,
1839 rq->rx_ring[0].size * rq->data_ring.desc_size,
1840 rq->data_ring.base, rq->data_ring.basePA);
1841 rq->data_ring.base = NULL;
1844 if (rq->comp_ring.base) {
1845 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1846 * sizeof(struct Vmxnet3_RxCompDesc),
1847 rq->comp_ring.base, rq->comp_ring.basePA);
1848 rq->comp_ring.base = NULL;
1851 kfree(rq->buf_info[0]);
1852 rq->buf_info[0] = NULL;
1853 rq->buf_info[1] = NULL;
1857 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
1861 for (i = 0; i < adapter->num_rx_queues; i++) {
1862 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1864 if (rq->data_ring.base) {
1865 dma_free_coherent(&adapter->pdev->dev,
1866 (rq->rx_ring[0].size *
1867 rq->data_ring.desc_size),
1869 rq->data_ring.basePA);
1870 rq->data_ring.base = NULL;
1871 rq->data_ring.desc_size = 0;
1877 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1878 struct vmxnet3_adapter *adapter)
1882 /* initialize buf_info */
1883 for (i = 0; i < rq->rx_ring[0].size; i++) {
1885 /* 1st buf for a pkt is skbuff */
1886 if (i % adapter->rx_buf_per_pkt == 0) {
1887 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1888 rq->buf_info[0][i].len = adapter->skb_buf_size;
1889 } else { /* subsequent bufs for a pkt is frag */
1890 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1891 rq->buf_info[0][i].len = PAGE_SIZE;
1894 for (i = 0; i < rq->rx_ring[1].size; i++) {
1895 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1896 rq->buf_info[1][i].len = PAGE_SIZE;
1899 /* reset internal state and allocate buffers for both rings */
1900 for (i = 0; i < 2; i++) {
1901 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1903 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1904 sizeof(struct Vmxnet3_RxDesc));
1905 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1906 rq->rx_ring[i].isOutOfOrder = 0;
1908 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1910 /* at least has 1 rx buffer for the 1st ring */
1913 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1915 /* reset the comp ring */
1916 rq->comp_ring.next2proc = 0;
1917 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1918 sizeof(struct Vmxnet3_RxCompDesc));
1919 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1922 rq->rx_ctx.skb = NULL;
1924 /* stats are not reset */
1930 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1934 for (i = 0; i < adapter->num_rx_queues; i++) {
1935 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1936 if (unlikely(err)) {
1937 dev_err(&adapter->netdev->dev, "%s: failed to "
1938 "initialize rx queue%i\n",
1939 adapter->netdev->name, i);
1949 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1953 struct vmxnet3_rx_buf_info *bi;
1955 for (i = 0; i < 2; i++) {
1957 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1958 rq->rx_ring[i].base = dma_alloc_coherent(
1959 &adapter->pdev->dev, sz,
1960 &rq->rx_ring[i].basePA,
1962 if (!rq->rx_ring[i].base) {
1963 netdev_err(adapter->netdev,
1964 "failed to allocate rx ring %d\n", i);
1969 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) {
1970 sz = rq->rx_ring[0].size * rq->data_ring.desc_size;
1971 rq->data_ring.base =
1972 dma_alloc_coherent(&adapter->pdev->dev, sz,
1973 &rq->data_ring.basePA,
1975 if (!rq->data_ring.base) {
1976 netdev_err(adapter->netdev,
1977 "rx data ring will be disabled\n");
1978 adapter->rxdataring_enabled = false;
1981 rq->data_ring.base = NULL;
1982 rq->data_ring.desc_size = 0;
1985 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1986 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1987 &rq->comp_ring.basePA,
1989 if (!rq->comp_ring.base) {
1990 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1994 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size,
1995 sizeof(rq->buf_info[0][0]), GFP_KERNEL,
1996 dev_to_node(&adapter->pdev->dev));
2000 rq->buf_info[0] = bi;
2001 rq->buf_info[1] = bi + rq->rx_ring[0].size;
2006 vmxnet3_rq_destroy(rq, adapter);
2012 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
2016 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
2018 for (i = 0; i < adapter->num_rx_queues; i++) {
2019 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
2020 if (unlikely(err)) {
2021 dev_err(&adapter->netdev->dev,
2022 "%s: failed to create rx queue%i\n",
2023 adapter->netdev->name, i);
2028 if (!adapter->rxdataring_enabled)
2029 vmxnet3_rq_destroy_all_rxdataring(adapter);
2033 vmxnet3_rq_destroy_all(adapter);
2038 /* Multiple queue aware polling function for tx and rx */
2041 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
2043 int rcd_done = 0, i;
2044 if (unlikely(adapter->shared->ecr))
2045 vmxnet3_process_events(adapter);
2046 for (i = 0; i < adapter->num_tx_queues; i++)
2047 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
2049 for (i = 0; i < adapter->num_rx_queues; i++)
2050 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
2057 vmxnet3_poll(struct napi_struct *napi, int budget)
2059 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
2060 struct vmxnet3_rx_queue, napi);
2063 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
2065 if (rxd_done < budget) {
2066 napi_complete_done(napi, rxd_done);
2067 vmxnet3_enable_all_intrs(rx_queue->adapter);
2073 * NAPI polling function for MSI-X mode with multiple Rx queues
2074 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
2078 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
2080 struct vmxnet3_rx_queue *rq = container_of(napi,
2081 struct vmxnet3_rx_queue, napi);
2082 struct vmxnet3_adapter *adapter = rq->adapter;
2085 /* When sharing interrupt with corresponding tx queue, process
2086 * tx completions in that queue as well
2088 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
2089 struct vmxnet3_tx_queue *tq =
2090 &adapter->tx_queue[rq - adapter->rx_queue];
2091 vmxnet3_tq_tx_complete(tq, adapter);
2094 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
2096 if (rxd_done < budget) {
2097 napi_complete_done(napi, rxd_done);
2098 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
2104 #ifdef CONFIG_PCI_MSI
2107 * Handle completion interrupts on tx queues
2108 * Returns whether or not the intr is handled
2112 vmxnet3_msix_tx(int irq, void *data)
2114 struct vmxnet3_tx_queue *tq = data;
2115 struct vmxnet3_adapter *adapter = tq->adapter;
2117 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2118 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
2120 /* Handle the case where only one irq is allocate for all tx queues */
2121 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2123 for (i = 0; i < adapter->num_tx_queues; i++) {
2124 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
2125 vmxnet3_tq_tx_complete(txq, adapter);
2128 vmxnet3_tq_tx_complete(tq, adapter);
2130 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
2137 * Handle completion interrupts on rx queues. Returns whether or not the
2142 vmxnet3_msix_rx(int irq, void *data)
2144 struct vmxnet3_rx_queue *rq = data;
2145 struct vmxnet3_adapter *adapter = rq->adapter;
2147 /* disable intr if needed */
2148 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2149 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
2150 napi_schedule(&rq->napi);
2156 *----------------------------------------------------------------------------
2158 * vmxnet3_msix_event --
2160 * vmxnet3 msix event intr handler
2163 * whether or not the intr is handled
2165 *----------------------------------------------------------------------------
2169 vmxnet3_msix_event(int irq, void *data)
2171 struct net_device *dev = data;
2172 struct vmxnet3_adapter *adapter = netdev_priv(dev);
2174 /* disable intr if needed */
2175 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2176 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
2178 if (adapter->shared->ecr)
2179 vmxnet3_process_events(adapter);
2181 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
2186 #endif /* CONFIG_PCI_MSI */
2189 /* Interrupt handler for vmxnet3 */
2191 vmxnet3_intr(int irq, void *dev_id)
2193 struct net_device *dev = dev_id;
2194 struct vmxnet3_adapter *adapter = netdev_priv(dev);
2196 if (adapter->intr.type == VMXNET3_IT_INTX) {
2197 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
2198 if (unlikely(icr == 0))
2204 /* disable intr if needed */
2205 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2206 vmxnet3_disable_all_intrs(adapter);
2208 napi_schedule(&adapter->rx_queue[0].napi);
2213 #ifdef CONFIG_NET_POLL_CONTROLLER
2215 /* netpoll callback. */
2217 vmxnet3_netpoll(struct net_device *netdev)
2219 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2221 switch (adapter->intr.type) {
2222 #ifdef CONFIG_PCI_MSI
2223 case VMXNET3_IT_MSIX: {
2225 for (i = 0; i < adapter->num_rx_queues; i++)
2226 vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
2230 case VMXNET3_IT_MSI:
2232 vmxnet3_intr(0, adapter->netdev);
2237 #endif /* CONFIG_NET_POLL_CONTROLLER */
2240 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
2242 struct vmxnet3_intr *intr = &adapter->intr;
2246 #ifdef CONFIG_PCI_MSI
2247 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2248 for (i = 0; i < adapter->num_tx_queues; i++) {
2249 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2250 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
2251 adapter->netdev->name, vector);
2253 intr->msix_entries[vector].vector,
2255 adapter->tx_queue[i].name,
2256 &adapter->tx_queue[i]);
2258 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
2259 adapter->netdev->name, vector);
2262 dev_err(&adapter->netdev->dev,
2263 "Failed to request irq for MSIX, %s, "
2265 adapter->tx_queue[i].name, err);
2269 /* Handle the case where only 1 MSIx was allocated for
2271 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2272 for (; i < adapter->num_tx_queues; i++)
2273 adapter->tx_queue[i].comp_ring.intr_idx
2278 adapter->tx_queue[i].comp_ring.intr_idx
2282 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
2285 for (i = 0; i < adapter->num_rx_queues; i++) {
2286 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
2287 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
2288 adapter->netdev->name, vector);
2290 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
2291 adapter->netdev->name, vector);
2292 err = request_irq(intr->msix_entries[vector].vector,
2294 adapter->rx_queue[i].name,
2295 &(adapter->rx_queue[i]));
2297 netdev_err(adapter->netdev,
2298 "Failed to request irq for MSIX, "
2300 adapter->rx_queue[i].name, err);
2304 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
2307 sprintf(intr->event_msi_vector_name, "%s-event-%d",
2308 adapter->netdev->name, vector);
2309 err = request_irq(intr->msix_entries[vector].vector,
2310 vmxnet3_msix_event, 0,
2311 intr->event_msi_vector_name, adapter->netdev);
2312 intr->event_intr_idx = vector;
2314 } else if (intr->type == VMXNET3_IT_MSI) {
2315 adapter->num_rx_queues = 1;
2316 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
2317 adapter->netdev->name, adapter->netdev);
2320 adapter->num_rx_queues = 1;
2321 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
2322 IRQF_SHARED, adapter->netdev->name,
2324 #ifdef CONFIG_PCI_MSI
2327 intr->num_intrs = vector + 1;
2329 netdev_err(adapter->netdev,
2330 "Failed to request irq (intr type:%d), error %d\n",
2333 /* Number of rx queues will not change after this */
2334 for (i = 0; i < adapter->num_rx_queues; i++) {
2335 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2337 rq->qid2 = i + adapter->num_rx_queues;
2338 rq->dataRingQid = i + 2 * adapter->num_rx_queues;
2341 /* init our intr settings */
2342 for (i = 0; i < intr->num_intrs; i++)
2343 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
2344 if (adapter->intr.type != VMXNET3_IT_MSIX) {
2345 adapter->intr.event_intr_idx = 0;
2346 for (i = 0; i < adapter->num_tx_queues; i++)
2347 adapter->tx_queue[i].comp_ring.intr_idx = 0;
2348 adapter->rx_queue[0].comp_ring.intr_idx = 0;
2351 netdev_info(adapter->netdev,
2352 "intr type %u, mode %u, %u vectors allocated\n",
2353 intr->type, intr->mask_mode, intr->num_intrs);
2361 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
2363 struct vmxnet3_intr *intr = &adapter->intr;
2364 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
2366 switch (intr->type) {
2367 #ifdef CONFIG_PCI_MSI
2368 case VMXNET3_IT_MSIX:
2372 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2373 for (i = 0; i < adapter->num_tx_queues; i++) {
2374 free_irq(intr->msix_entries[vector++].vector,
2375 &(adapter->tx_queue[i]));
2376 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
2381 for (i = 0; i < adapter->num_rx_queues; i++) {
2382 free_irq(intr->msix_entries[vector++].vector,
2383 &(adapter->rx_queue[i]));
2386 free_irq(intr->msix_entries[vector].vector,
2388 BUG_ON(vector >= intr->num_intrs);
2392 case VMXNET3_IT_MSI:
2393 free_irq(adapter->pdev->irq, adapter->netdev);
2395 case VMXNET3_IT_INTX:
2396 free_irq(adapter->pdev->irq, adapter->netdev);
2405 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
2407 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2410 /* allow untagged pkts */
2411 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
2413 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2414 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2419 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
2421 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2423 if (!(netdev->flags & IFF_PROMISC)) {
2424 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2425 unsigned long flags;
2427 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2428 spin_lock_irqsave(&adapter->cmd_lock, flags);
2429 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2430 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2431 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2434 set_bit(vid, adapter->active_vlans);
2441 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
2443 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2445 if (!(netdev->flags & IFF_PROMISC)) {
2446 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2447 unsigned long flags;
2449 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
2450 spin_lock_irqsave(&adapter->cmd_lock, flags);
2451 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2452 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2453 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2456 clear_bit(vid, adapter->active_vlans);
2463 vmxnet3_copy_mc(struct net_device *netdev)
2466 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2468 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2470 /* We may be called with BH disabled */
2471 buf = kmalloc(sz, GFP_ATOMIC);
2473 struct netdev_hw_addr *ha;
2476 netdev_for_each_mc_addr(ha, netdev)
2477 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2486 vmxnet3_set_mc(struct net_device *netdev)
2488 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2489 unsigned long flags;
2490 struct Vmxnet3_RxFilterConf *rxConf =
2491 &adapter->shared->devRead.rxFilterConf;
2492 u8 *new_table = NULL;
2493 dma_addr_t new_table_pa = 0;
2494 bool new_table_pa_valid = false;
2495 u32 new_mode = VMXNET3_RXM_UCAST;
2497 if (netdev->flags & IFF_PROMISC) {
2498 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2499 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2501 new_mode |= VMXNET3_RXM_PROMISC;
2503 vmxnet3_restore_vlan(adapter);
2506 if (netdev->flags & IFF_BROADCAST)
2507 new_mode |= VMXNET3_RXM_BCAST;
2509 if (netdev->flags & IFF_ALLMULTI)
2510 new_mode |= VMXNET3_RXM_ALL_MULTI;
2512 if (!netdev_mc_empty(netdev)) {
2513 new_table = vmxnet3_copy_mc(netdev);
2515 size_t sz = netdev_mc_count(netdev) * ETH_ALEN;
2517 rxConf->mfTableLen = cpu_to_le16(sz);
2518 new_table_pa = dma_map_single(
2519 &adapter->pdev->dev,
2523 if (!dma_mapping_error(&adapter->pdev->dev,
2525 new_mode |= VMXNET3_RXM_MCAST;
2526 new_table_pa_valid = true;
2527 rxConf->mfTablePA = cpu_to_le64(
2531 if (!new_table_pa_valid) {
2533 "failed to copy mcast list, setting ALL_MULTI\n");
2534 new_mode |= VMXNET3_RXM_ALL_MULTI;
2538 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2539 rxConf->mfTableLen = 0;
2540 rxConf->mfTablePA = 0;
2543 spin_lock_irqsave(&adapter->cmd_lock, flags);
2544 if (new_mode != rxConf->rxMode) {
2545 rxConf->rxMode = cpu_to_le32(new_mode);
2546 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2547 VMXNET3_CMD_UPDATE_RX_MODE);
2548 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2549 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2552 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2553 VMXNET3_CMD_UPDATE_MAC_FILTERS);
2554 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2556 if (new_table_pa_valid)
2557 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2558 rxConf->mfTableLen, DMA_TO_DEVICE);
2563 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2567 for (i = 0; i < adapter->num_rx_queues; i++)
2568 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2573 * Set up driver_shared based on settings in adapter.
2577 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2579 struct Vmxnet3_DriverShared *shared = adapter->shared;
2580 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2581 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt;
2582 struct Vmxnet3_TxQueueConf *tqc;
2583 struct Vmxnet3_RxQueueConf *rqc;
2586 memset(shared, 0, sizeof(*shared));
2588 /* driver settings */
2589 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2590 devRead->misc.driverInfo.version = cpu_to_le32(
2591 VMXNET3_DRIVER_VERSION_NUM);
2592 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2593 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2594 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2595 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2596 *((u32 *)&devRead->misc.driverInfo.gos));
2597 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2598 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2600 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2601 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2603 /* set up feature flags */
2604 if (adapter->netdev->features & NETIF_F_RXCSUM)
2605 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2607 if (adapter->netdev->features & NETIF_F_LRO) {
2608 devRead->misc.uptFeatures |= UPT1_F_LRO;
2609 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2611 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2612 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2614 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL |
2615 NETIF_F_GSO_UDP_TUNNEL_CSUM))
2616 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD;
2618 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2619 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2620 devRead->misc.queueDescLen = cpu_to_le32(
2621 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2622 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2624 /* tx queue settings */
2625 devRead->misc.numTxQueues = adapter->num_tx_queues;
2626 for (i = 0; i < adapter->num_tx_queues; i++) {
2627 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2628 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2629 tqc = &adapter->tqd_start[i].conf;
2630 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2631 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2632 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2633 tqc->ddPA = cpu_to_le64(~0ULL);
2634 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2635 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2636 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);
2637 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2638 tqc->ddLen = cpu_to_le32(0);
2639 tqc->intrIdx = tq->comp_ring.intr_idx;
2642 /* rx queue settings */
2643 devRead->misc.numRxQueues = adapter->num_rx_queues;
2644 for (i = 0; i < adapter->num_rx_queues; i++) {
2645 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2646 rqc = &adapter->rqd_start[i].conf;
2647 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2648 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2649 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2650 rqc->ddPA = cpu_to_le64(~0ULL);
2651 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2652 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2653 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2654 rqc->ddLen = cpu_to_le32(0);
2655 rqc->intrIdx = rq->comp_ring.intr_idx;
2656 if (VMXNET3_VERSION_GE_3(adapter)) {
2657 rqc->rxDataRingBasePA =
2658 cpu_to_le64(rq->data_ring.basePA);
2659 rqc->rxDataRingDescSize =
2660 cpu_to_le16(rq->data_ring.desc_size);
2665 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2668 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2670 devRead->misc.uptFeatures |= UPT1_F_RSS;
2671 devRead->misc.numRxQueues = adapter->num_rx_queues;
2672 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2673 UPT1_RSS_HASH_TYPE_IPV4 |
2674 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2675 UPT1_RSS_HASH_TYPE_IPV6;
2676 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2677 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2678 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2679 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey));
2681 for (i = 0; i < rssConf->indTableSize; i++)
2682 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2683 i, adapter->num_rx_queues);
2685 devRead->rssConfDesc.confVer = 1;
2686 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2687 devRead->rssConfDesc.confPA =
2688 cpu_to_le64(adapter->rss_conf_pa);
2691 #endif /* VMXNET3_RSS */
2694 if (!VMXNET3_VERSION_GE_6(adapter) ||
2695 !adapter->queuesExtEnabled) {
2696 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2698 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2699 for (i = 0; i < adapter->intr.num_intrs; i++)
2700 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2702 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2703 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2705 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode ==
2707 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs;
2708 for (i = 0; i < adapter->intr.num_intrs; i++)
2709 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i];
2711 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx;
2712 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2715 /* rx filter settings */
2716 devRead->rxFilterConf.rxMode = 0;
2717 vmxnet3_restore_vlan(adapter);
2718 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2720 /* the rest are already zeroed */
2724 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter)
2726 struct Vmxnet3_DriverShared *shared = adapter->shared;
2727 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2728 unsigned long flags;
2730 if (!VMXNET3_VERSION_GE_7(adapter))
2733 cmdInfo->ringBufSize = adapter->ringBufSize;
2734 spin_lock_irqsave(&adapter->cmd_lock, flags);
2735 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2736 VMXNET3_CMD_SET_RING_BUFFER_SIZE);
2737 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2741 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
2743 struct Vmxnet3_DriverShared *shared = adapter->shared;
2744 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2745 unsigned long flags;
2747 if (!VMXNET3_VERSION_GE_3(adapter))
2750 spin_lock_irqsave(&adapter->cmd_lock, flags);
2751 cmdInfo->varConf.confVer = 1;
2752 cmdInfo->varConf.confLen =
2753 cpu_to_le32(sizeof(*adapter->coal_conf));
2754 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa);
2756 if (adapter->default_coal_mode) {
2757 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2758 VMXNET3_CMD_GET_COALESCE);
2760 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2761 VMXNET3_CMD_SET_COALESCE);
2764 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2768 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter)
2770 struct Vmxnet3_DriverShared *shared = adapter->shared;
2771 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2772 unsigned long flags;
2774 if (!VMXNET3_VERSION_GE_4(adapter))
2777 spin_lock_irqsave(&adapter->cmd_lock, flags);
2779 if (adapter->default_rss_fields) {
2780 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2781 VMXNET3_CMD_GET_RSS_FIELDS);
2782 adapter->rss_fields =
2783 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2785 if (VMXNET3_VERSION_GE_7(adapter)) {
2786 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 ||
2787 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) &&
2788 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2789 VMXNET3_CAP_UDP_RSS)) {
2790 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS;
2792 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS);
2795 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) &&
2796 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2797 VMXNET3_CAP_ESP_RSS_IPV4)) {
2798 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4;
2800 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4);
2803 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) &&
2804 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2805 VMXNET3_CAP_ESP_RSS_IPV6)) {
2806 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6;
2808 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6);
2811 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
2812 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
2813 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2815 cmdInfo->setRssFields = adapter->rss_fields;
2816 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2817 VMXNET3_CMD_SET_RSS_FIELDS);
2818 /* Not all requested RSS may get applied, so get and
2819 * cache what was actually applied.
2821 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2822 VMXNET3_CMD_GET_RSS_FIELDS);
2823 adapter->rss_fields =
2824 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2827 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2831 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2835 unsigned long flags;
2837 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2838 " ring sizes %u %u %u\n", adapter->netdev->name,
2839 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2840 adapter->tx_queue[0].tx_ring.size,
2841 adapter->rx_queue[0].rx_ring[0].size,
2842 adapter->rx_queue[0].rx_ring[1].size);
2844 vmxnet3_tq_init_all(adapter);
2845 err = vmxnet3_rq_init_all(adapter);
2847 netdev_err(adapter->netdev,
2848 "Failed to init rx queue error %d\n", err);
2852 err = vmxnet3_request_irqs(adapter);
2854 netdev_err(adapter->netdev,
2855 "Failed to setup irq for error %d\n", err);
2859 vmxnet3_setup_driver_shared(adapter);
2861 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2862 adapter->shared_pa));
2863 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2864 adapter->shared_pa));
2865 spin_lock_irqsave(&adapter->cmd_lock, flags);
2866 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2867 VMXNET3_CMD_ACTIVATE_DEV);
2868 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2869 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2872 netdev_err(adapter->netdev,
2873 "Failed to activate dev: error %u\n", ret);
2878 vmxnet3_init_bufsize(adapter);
2879 vmxnet3_init_coalesce(adapter);
2880 vmxnet3_init_rssfields(adapter);
2882 for (i = 0; i < adapter->num_rx_queues; i++) {
2883 VMXNET3_WRITE_BAR0_REG(adapter,
2884 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN,
2885 adapter->rx_queue[i].rx_ring[0].next2fill);
2886 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset +
2887 (i * VMXNET3_REG_ALIGN)),
2888 adapter->rx_queue[i].rx_ring[1].next2fill);
2891 /* Apply the rx filter settins last. */
2892 vmxnet3_set_mc(adapter->netdev);
2895 * Check link state when first activating device. It will start the
2896 * tx queue if the link is up.
2898 vmxnet3_check_link(adapter, true);
2899 netif_tx_wake_all_queues(adapter->netdev);
2900 for (i = 0; i < adapter->num_rx_queues; i++)
2901 napi_enable(&adapter->rx_queue[i].napi);
2902 vmxnet3_enable_all_intrs(adapter);
2903 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2907 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2908 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2909 vmxnet3_free_irqs(adapter);
2912 /* free up buffers we allocated */
2913 vmxnet3_rq_cleanup_all(adapter);
2919 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2921 unsigned long flags;
2922 spin_lock_irqsave(&adapter->cmd_lock, flags);
2923 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2924 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2929 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2932 unsigned long flags;
2933 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2937 spin_lock_irqsave(&adapter->cmd_lock, flags);
2938 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2939 VMXNET3_CMD_QUIESCE_DEV);
2940 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2941 vmxnet3_disable_all_intrs(adapter);
2943 for (i = 0; i < adapter->num_rx_queues; i++)
2944 napi_disable(&adapter->rx_queue[i].napi);
2945 netif_tx_disable(adapter->netdev);
2946 adapter->link_speed = 0;
2947 netif_carrier_off(adapter->netdev);
2949 vmxnet3_tq_cleanup_all(adapter);
2950 vmxnet3_rq_cleanup_all(adapter);
2951 vmxnet3_free_irqs(adapter);
2957 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac)
2962 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2964 tmp = (mac[5] << 8) | mac[4];
2965 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2970 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2972 struct sockaddr *addr = p;
2973 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2975 dev_addr_set(netdev, addr->sa_data);
2976 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2982 /* ==================== initialization and cleanup routines ============ */
2985 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter)
2988 unsigned long mmio_start, mmio_len;
2989 struct pci_dev *pdev = adapter->pdev;
2991 err = pci_enable_device(pdev);
2993 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2997 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2998 vmxnet3_driver_name);
3001 "Failed to request region for adapter: error %d\n", err);
3002 goto err_enable_device;
3005 pci_set_master(pdev);
3007 mmio_start = pci_resource_start(pdev, 0);
3008 mmio_len = pci_resource_len(pdev, 0);
3009 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
3010 if (!adapter->hw_addr0) {
3011 dev_err(&pdev->dev, "Failed to map bar0\n");
3016 mmio_start = pci_resource_start(pdev, 1);
3017 mmio_len = pci_resource_len(pdev, 1);
3018 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
3019 if (!adapter->hw_addr1) {
3020 dev_err(&pdev->dev, "Failed to map bar1\n");
3027 iounmap(adapter->hw_addr0);
3029 pci_release_selected_regions(pdev, (1 << 2) - 1);
3031 pci_disable_device(pdev);
3037 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
3039 BUG_ON(!adapter->pdev);
3041 iounmap(adapter->hw_addr0);
3042 iounmap(adapter->hw_addr1);
3043 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
3044 pci_disable_device(adapter->pdev);
3049 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
3051 size_t sz, i, ring0_size, ring1_size, comp_size;
3052 /* With version7 ring1 will have only T0 buffers */
3053 if (!VMXNET3_VERSION_GE_7(adapter)) {
3054 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
3055 VMXNET3_MAX_ETH_HDR_SIZE) {
3056 adapter->skb_buf_size = adapter->netdev->mtu +
3057 VMXNET3_MAX_ETH_HDR_SIZE;
3058 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
3059 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
3061 adapter->rx_buf_per_pkt = 1;
3063 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
3064 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
3065 VMXNET3_MAX_ETH_HDR_SIZE;
3066 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
3069 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE,
3070 VMXNET3_MAX_SKB_BUF_SIZE);
3071 adapter->rx_buf_per_pkt = 1;
3072 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size);
3073 adapter->ringBufSize.ring1BufSizeType1 = 0;
3074 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE);
3078 * for simplicity, force the ring0 size to be a multiple of
3079 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
3081 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
3082 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
3083 ring0_size = (ring0_size + sz - 1) / sz * sz;
3084 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
3086 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
3087 ring1_size = (ring1_size + sz - 1) / sz * sz;
3088 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE /
3090 /* For v7 and later, keep ring size power of 2 for UPT */
3091 if (VMXNET3_VERSION_GE_7(adapter)) {
3092 ring0_size = rounddown_pow_of_two(ring0_size);
3093 ring1_size = rounddown_pow_of_two(ring1_size);
3095 comp_size = ring0_size + ring1_size;
3097 for (i = 0; i < adapter->num_rx_queues; i++) {
3098 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3100 rq->rx_ring[0].size = ring0_size;
3101 rq->rx_ring[1].size = ring1_size;
3102 rq->comp_ring.size = comp_size;
3108 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
3109 u32 rx_ring_size, u32 rx_ring2_size,
3110 u16 txdata_desc_size, u16 rxdata_desc_size)
3114 for (i = 0; i < adapter->num_tx_queues; i++) {
3115 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
3116 tq->tx_ring.size = tx_ring_size;
3117 tq->data_ring.size = tx_ring_size;
3118 tq->comp_ring.size = tx_ring_size;
3119 tq->txdata_desc_size = txdata_desc_size;
3120 tq->shared = &adapter->tqd_start[i].ctrl;
3122 tq->adapter = adapter;
3124 err = vmxnet3_tq_create(tq, adapter);
3126 * Too late to change num_tx_queues. We cannot do away with
3127 * lesser number of queues than what we asked for
3133 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
3134 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
3135 vmxnet3_adjust_rx_ring_size(adapter);
3137 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
3138 for (i = 0; i < adapter->num_rx_queues; i++) {
3139 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3140 /* qid and qid2 for rx queues will be assigned later when num
3141 * of rx queues is finalized after allocating intrs */
3142 rq->shared = &adapter->rqd_start[i].ctrl;
3143 rq->adapter = adapter;
3144 rq->data_ring.desc_size = rxdata_desc_size;
3145 err = vmxnet3_rq_create(rq, adapter);
3148 netdev_err(adapter->netdev,
3149 "Could not allocate any rx queues. "
3153 netdev_info(adapter->netdev,
3154 "Number of rx queues changed "
3156 adapter->num_rx_queues = i;
3163 if (!adapter->rxdataring_enabled)
3164 vmxnet3_rq_destroy_all_rxdataring(adapter);
3168 vmxnet3_tq_destroy_all(adapter);
3173 vmxnet3_open(struct net_device *netdev)
3175 struct vmxnet3_adapter *adapter;
3178 adapter = netdev_priv(netdev);
3180 for (i = 0; i < adapter->num_tx_queues; i++)
3181 spin_lock_init(&adapter->tx_queue[i].tx_lock);
3183 if (VMXNET3_VERSION_GE_3(adapter)) {
3184 unsigned long flags;
3185 u16 txdata_desc_size;
3187 spin_lock_irqsave(&adapter->cmd_lock, flags);
3188 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3189 VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
3190 txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter,
3192 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3194 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) ||
3195 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) ||
3196 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) {
3197 adapter->txdata_desc_size =
3198 sizeof(struct Vmxnet3_TxDataDesc);
3200 adapter->txdata_desc_size = txdata_desc_size;
3203 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc);
3206 err = vmxnet3_create_queues(adapter,
3207 adapter->tx_ring_size,
3208 adapter->rx_ring_size,
3209 adapter->rx_ring2_size,
3210 adapter->txdata_desc_size,
3211 adapter->rxdata_desc_size);
3215 err = vmxnet3_activate_dev(adapter);
3222 vmxnet3_rq_destroy_all(adapter);
3223 vmxnet3_tq_destroy_all(adapter);
3230 vmxnet3_close(struct net_device *netdev)
3232 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3235 * Reset_work may be in the middle of resetting the device, wait for its
3238 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3239 usleep_range(1000, 2000);
3241 vmxnet3_quiesce_dev(adapter);
3243 vmxnet3_rq_destroy_all(adapter);
3244 vmxnet3_tq_destroy_all(adapter);
3246 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3254 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
3259 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
3260 * vmxnet3_close() will deadlock.
3262 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
3264 /* we need to enable NAPI, otherwise dev_close will deadlock */
3265 for (i = 0; i < adapter->num_rx_queues; i++)
3266 napi_enable(&adapter->rx_queue[i].napi);
3268 * Need to clear the quiesce bit to ensure that vmxnet3_close
3269 * can quiesce the device properly
3271 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3272 dev_close(adapter->netdev);
3277 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
3279 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3282 netdev->mtu = new_mtu;
3285 * Reset_work may be in the middle of resetting the device, wait for its
3288 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3289 usleep_range(1000, 2000);
3291 if (netif_running(netdev)) {
3292 vmxnet3_quiesce_dev(adapter);
3293 vmxnet3_reset_dev(adapter);
3295 /* we need to re-create the rx queue based on the new mtu */
3296 vmxnet3_rq_destroy_all(adapter);
3297 vmxnet3_adjust_rx_ring_size(adapter);
3298 err = vmxnet3_rq_create_all(adapter);
3301 "failed to re-create rx queues, "
3302 " error %d. Closing it.\n", err);
3306 err = vmxnet3_activate_dev(adapter);
3309 "failed to re-activate, error %d. "
3310 "Closing it\n", err);
3316 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3318 vmxnet3_force_close(adapter);
3325 vmxnet3_declare_features(struct vmxnet3_adapter *adapter)
3327 struct net_device *netdev = adapter->netdev;
3329 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
3330 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3331 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3332 NETIF_F_LRO | NETIF_F_HIGHDMA;
3334 if (VMXNET3_VERSION_GE_4(adapter)) {
3335 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
3336 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3338 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM |
3339 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3340 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3341 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL |
3342 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3345 if (VMXNET3_VERSION_GE_7(adapter)) {
3346 unsigned long flags;
3348 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3349 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) {
3350 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD;
3352 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3353 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) {
3354 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD;
3356 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3357 VMXNET3_CAP_GENEVE_TSO)) {
3358 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO;
3360 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3361 VMXNET3_CAP_VXLAN_TSO)) {
3362 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO;
3364 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3365 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) {
3366 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD;
3368 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3369 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) {
3370 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD;
3373 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3374 spin_lock_irqsave(&adapter->cmd_lock, flags);
3375 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3376 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3377 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3379 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) &&
3380 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) &&
3381 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) &&
3382 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) {
3383 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3384 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3386 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) &&
3387 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) {
3388 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3389 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3393 netdev->vlan_features = netdev->hw_features &
3394 ~(NETIF_F_HW_VLAN_CTAG_TX |
3395 NETIF_F_HW_VLAN_CTAG_RX);
3396 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
3401 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
3405 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
3408 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
3409 mac[4] = tmp & 0xff;
3410 mac[5] = (tmp >> 8) & 0xff;
3413 #ifdef CONFIG_PCI_MSI
3416 * Enable MSIx vectors.
3418 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
3420 * number of vectors which were enabled otherwise (this number is greater
3421 * than VMXNET3_LINUX_MIN_MSIX_VECT)
3425 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
3427 int ret = pci_enable_msix_range(adapter->pdev,
3428 adapter->intr.msix_entries, nvec, nvec);
3430 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
3431 dev_err(&adapter->netdev->dev,
3432 "Failed to enable %d MSI-X, trying %d\n",
3433 nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
3435 ret = pci_enable_msix_range(adapter->pdev,
3436 adapter->intr.msix_entries,
3437 VMXNET3_LINUX_MIN_MSIX_VECT,
3438 VMXNET3_LINUX_MIN_MSIX_VECT);
3442 dev_err(&adapter->netdev->dev,
3443 "Failed to enable MSI-X, error: %d\n", ret);
3450 #endif /* CONFIG_PCI_MSI */
3453 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
3456 unsigned long flags;
3459 spin_lock_irqsave(&adapter->cmd_lock, flags);
3460 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3461 VMXNET3_CMD_GET_CONF_INTR);
3462 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3463 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3464 adapter->intr.type = cfg & 0x3;
3465 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
3467 if (adapter->intr.type == VMXNET3_IT_AUTO) {
3468 adapter->intr.type = VMXNET3_IT_MSIX;
3471 #ifdef CONFIG_PCI_MSI
3472 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3473 int i, nvec, nvec_allocated;
3475 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
3476 1 : adapter->num_tx_queues;
3477 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
3478 0 : adapter->num_rx_queues;
3479 nvec += 1; /* for link event */
3480 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
3481 nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
3483 for (i = 0; i < nvec; i++)
3484 adapter->intr.msix_entries[i].entry = i;
3486 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec);
3487 if (nvec_allocated < 0)
3490 /* If we cannot allocate one MSIx vector per queue
3491 * then limit the number of rx queues to 1
3493 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT &&
3494 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) {
3495 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
3496 || adapter->num_rx_queues != 1) {
3497 adapter->share_intr = VMXNET3_INTR_TXSHARE;
3498 netdev_err(adapter->netdev,
3499 "Number of rx queues : 1\n");
3500 adapter->num_rx_queues = 1;
3504 adapter->intr.num_intrs = nvec_allocated;
3508 /* If we cannot allocate MSIx vectors use only one rx queue */
3509 dev_info(&adapter->pdev->dev,
3510 "Failed to enable MSI-X, error %d. "
3511 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated);
3513 adapter->intr.type = VMXNET3_IT_MSI;
3516 if (adapter->intr.type == VMXNET3_IT_MSI) {
3517 if (!pci_enable_msi(adapter->pdev)) {
3518 adapter->num_rx_queues = 1;
3519 adapter->intr.num_intrs = 1;
3523 #endif /* CONFIG_PCI_MSI */
3525 adapter->num_rx_queues = 1;
3526 dev_info(&adapter->netdev->dev,
3527 "Using INTx interrupt, #Rx queues: 1.\n");
3528 adapter->intr.type = VMXNET3_IT_INTX;
3530 /* INT-X related setting */
3531 adapter->intr.num_intrs = 1;
3536 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
3538 if (adapter->intr.type == VMXNET3_IT_MSIX)
3539 pci_disable_msix(adapter->pdev);
3540 else if (adapter->intr.type == VMXNET3_IT_MSI)
3541 pci_disable_msi(adapter->pdev);
3543 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
3548 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue)
3550 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3551 adapter->tx_timeout_count++;
3553 netdev_err(adapter->netdev, "tx hang\n");
3554 schedule_work(&adapter->work);
3559 vmxnet3_reset_work(struct work_struct *data)
3561 struct vmxnet3_adapter *adapter;
3563 adapter = container_of(data, struct vmxnet3_adapter, work);
3565 /* if another thread is resetting the device, no need to proceed */
3566 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3569 /* if the device is closed, we must leave it alone */
3571 if (netif_running(adapter->netdev)) {
3572 netdev_notice(adapter->netdev, "resetting\n");
3573 vmxnet3_quiesce_dev(adapter);
3574 vmxnet3_reset_dev(adapter);
3575 vmxnet3_activate_dev(adapter);
3577 netdev_info(adapter->netdev, "already closed\n");
3581 netif_wake_queue(adapter->netdev);
3582 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3587 vmxnet3_probe_device(struct pci_dev *pdev,
3588 const struct pci_device_id *id)
3590 static const struct net_device_ops vmxnet3_netdev_ops = {
3591 .ndo_open = vmxnet3_open,
3592 .ndo_stop = vmxnet3_close,
3593 .ndo_start_xmit = vmxnet3_xmit_frame,
3594 .ndo_set_mac_address = vmxnet3_set_mac_addr,
3595 .ndo_change_mtu = vmxnet3_change_mtu,
3596 .ndo_fix_features = vmxnet3_fix_features,
3597 .ndo_set_features = vmxnet3_set_features,
3598 .ndo_features_check = vmxnet3_features_check,
3599 .ndo_get_stats64 = vmxnet3_get_stats64,
3600 .ndo_tx_timeout = vmxnet3_tx_timeout,
3601 .ndo_set_rx_mode = vmxnet3_set_mc,
3602 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
3603 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
3604 #ifdef CONFIG_NET_POLL_CONTROLLER
3605 .ndo_poll_controller = vmxnet3_netpoll,
3610 struct net_device *netdev;
3611 struct vmxnet3_adapter *adapter;
3617 unsigned long flags;
3619 if (!pci_msi_enabled())
3624 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3625 (int)num_online_cpus());
3631 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
3632 (int)num_online_cpus());
3636 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
3637 max(num_tx_queues, num_rx_queues));
3641 pci_set_drvdata(pdev, netdev);
3642 adapter = netdev_priv(netdev);
3643 adapter->netdev = netdev;
3644 adapter->pdev = pdev;
3646 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
3647 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
3648 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
3650 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3652 dev_err(&pdev->dev, "dma_set_mask failed\n");
3656 spin_lock_init(&adapter->cmd_lock);
3657 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
3658 sizeof(struct vmxnet3_adapter),
3660 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) {
3661 dev_err(&pdev->dev, "Failed to map dma\n");
3665 adapter->shared = dma_alloc_coherent(
3666 &adapter->pdev->dev,
3667 sizeof(struct Vmxnet3_DriverShared),
3668 &adapter->shared_pa, GFP_KERNEL);
3669 if (!adapter->shared) {
3670 dev_err(&pdev->dev, "Failed to allocate memory\n");
3672 goto err_alloc_shared;
3675 err = vmxnet3_alloc_pci_resources(adapter);
3679 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3680 if (ver & (1 << VMXNET3_REV_7)) {
3681 VMXNET3_WRITE_BAR1_REG(adapter,
3683 1 << VMXNET3_REV_7);
3684 adapter->version = VMXNET3_REV_7 + 1;
3685 } else if (ver & (1 << VMXNET3_REV_6)) {
3686 VMXNET3_WRITE_BAR1_REG(adapter,
3688 1 << VMXNET3_REV_6);
3689 adapter->version = VMXNET3_REV_6 + 1;
3690 } else if (ver & (1 << VMXNET3_REV_5)) {
3691 VMXNET3_WRITE_BAR1_REG(adapter,
3693 1 << VMXNET3_REV_5);
3694 adapter->version = VMXNET3_REV_5 + 1;
3695 } else if (ver & (1 << VMXNET3_REV_4)) {
3696 VMXNET3_WRITE_BAR1_REG(adapter,
3698 1 << VMXNET3_REV_4);
3699 adapter->version = VMXNET3_REV_4 + 1;
3700 } else if (ver & (1 << VMXNET3_REV_3)) {
3701 VMXNET3_WRITE_BAR1_REG(adapter,
3703 1 << VMXNET3_REV_3);
3704 adapter->version = VMXNET3_REV_3 + 1;
3705 } else if (ver & (1 << VMXNET3_REV_2)) {
3706 VMXNET3_WRITE_BAR1_REG(adapter,
3708 1 << VMXNET3_REV_2);
3709 adapter->version = VMXNET3_REV_2 + 1;
3710 } else if (ver & (1 << VMXNET3_REV_1)) {
3711 VMXNET3_WRITE_BAR1_REG(adapter,
3713 1 << VMXNET3_REV_1);
3714 adapter->version = VMXNET3_REV_1 + 1;
3717 "Incompatible h/w version (0x%x) for adapter\n", ver);
3721 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version);
3723 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3725 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3728 "Incompatible upt version (0x%x) for adapter\n", ver);
3733 if (VMXNET3_VERSION_GE_7(adapter)) {
3734 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR);
3735 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR);
3736 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3737 adapter->dev_caps[0] = adapter->devcap_supported[0] &
3738 (1UL << VMXNET3_CAP_LARGE_BAR);
3740 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) &&
3741 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
3742 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
3743 adapter->dev_caps[0] |= adapter->devcap_supported[0] &
3744 (1UL << VMXNET3_CAP_OOORX_COMP);
3746 if (adapter->dev_caps[0])
3747 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3749 spin_lock_irqsave(&adapter->cmd_lock, flags);
3750 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3751 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3752 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3755 if (VMXNET3_VERSION_GE_7(adapter) &&
3756 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3757 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD;
3758 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD;
3759 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2;
3761 adapter->tx_prod_offset = VMXNET3_REG_TXPROD;
3762 adapter->rx_prod_offset = VMXNET3_REG_RXPROD;
3763 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2;
3766 if (VMXNET3_VERSION_GE_6(adapter)) {
3767 spin_lock_irqsave(&adapter->cmd_lock, flags);
3768 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3769 VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3770 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3771 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3773 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff));
3774 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff));
3776 adapter->num_rx_queues = min(num_rx_queues,
3777 VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3778 adapter->num_tx_queues = min(num_tx_queues,
3779 VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3781 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES ||
3782 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) {
3783 adapter->queuesExtEnabled = true;
3785 adapter->queuesExtEnabled = false;
3788 adapter->queuesExtEnabled = false;
3789 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3790 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
3791 adapter->num_rx_queues = min(num_rx_queues,
3792 VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3793 adapter->num_tx_queues = min(num_tx_queues,
3794 VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3796 dev_info(&pdev->dev,
3797 "# of Tx queues : %d, # of Rx queues : %d\n",
3798 adapter->num_tx_queues, adapter->num_rx_queues);
3800 adapter->rx_buf_per_pkt = 1;
3802 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3803 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3804 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3805 &adapter->queue_desc_pa,
3808 if (!adapter->tqd_start) {
3809 dev_err(&pdev->dev, "Failed to allocate memory\n");
3813 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3814 adapter->num_tx_queues);
3816 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3817 sizeof(struct Vmxnet3_PMConf),
3818 &adapter->pm_conf_pa,
3820 if (adapter->pm_conf == NULL) {
3827 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3828 sizeof(struct UPT1_RSSConf),
3829 &adapter->rss_conf_pa,
3831 if (adapter->rss_conf == NULL) {
3835 #endif /* VMXNET3_RSS */
3837 if (VMXNET3_VERSION_GE_3(adapter)) {
3838 adapter->coal_conf =
3839 dma_alloc_coherent(&adapter->pdev->dev,
3840 sizeof(struct Vmxnet3_CoalesceScheme)
3842 &adapter->coal_conf_pa,
3844 if (!adapter->coal_conf) {
3848 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
3849 adapter->default_coal_mode = true;
3852 if (VMXNET3_VERSION_GE_4(adapter)) {
3853 adapter->default_rss_fields = true;
3854 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT;
3857 SET_NETDEV_DEV(netdev, &pdev->dev);
3858 vmxnet3_declare_features(adapter);
3860 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
3861 VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
3863 if (adapter->num_tx_queues == adapter->num_rx_queues)
3864 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3866 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3868 vmxnet3_alloc_intr_resources(adapter);
3871 if (adapter->num_rx_queues > 1 &&
3872 adapter->intr.type == VMXNET3_IT_MSIX) {
3873 adapter->rss = true;
3874 netdev->hw_features |= NETIF_F_RXHASH;
3875 netdev->features |= NETIF_F_RXHASH;
3876 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3878 adapter->rss = false;
3882 vmxnet3_read_mac_addr(adapter, mac);
3883 dev_addr_set(netdev, mac);
3885 netdev->netdev_ops = &vmxnet3_netdev_ops;
3886 vmxnet3_set_ethtool_ops(netdev);
3887 netdev->watchdog_timeo = 5 * HZ;
3889 /* MTU range: 60 - 9190 */
3890 netdev->min_mtu = VMXNET3_MIN_MTU;
3891 if (VMXNET3_VERSION_GE_6(adapter))
3892 netdev->max_mtu = VMXNET3_V6_MAX_MTU;
3894 netdev->max_mtu = VMXNET3_MAX_MTU;
3896 INIT_WORK(&adapter->work, vmxnet3_reset_work);
3897 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3899 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3901 for (i = 0; i < adapter->num_rx_queues; i++) {
3902 netif_napi_add(adapter->netdev,
3903 &adapter->rx_queue[i].napi,
3904 vmxnet3_poll_rx_only);
3907 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3911 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3912 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3914 netif_carrier_off(netdev);
3915 err = register_netdev(netdev);
3918 dev_err(&pdev->dev, "Failed to register adapter\n");
3922 vmxnet3_check_link(adapter, false);
3926 if (VMXNET3_VERSION_GE_3(adapter)) {
3927 dma_free_coherent(&adapter->pdev->dev,
3928 sizeof(struct Vmxnet3_CoalesceScheme),
3929 adapter->coal_conf, adapter->coal_conf_pa);
3931 vmxnet3_free_intr_resources(adapter);
3934 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3935 adapter->rss_conf, adapter->rss_conf_pa);
3938 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3939 adapter->pm_conf, adapter->pm_conf_pa);
3941 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3942 adapter->queue_desc_pa);
3944 vmxnet3_free_pci_resources(adapter);
3946 dma_free_coherent(&adapter->pdev->dev,
3947 sizeof(struct Vmxnet3_DriverShared),
3948 adapter->shared, adapter->shared_pa);
3950 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3951 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
3953 free_netdev(netdev);
3959 vmxnet3_remove_device(struct pci_dev *pdev)
3961 struct net_device *netdev = pci_get_drvdata(pdev);
3962 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3964 int num_rx_queues, rx_queues;
3965 unsigned long flags;
3969 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3970 (int)num_online_cpus());
3974 if (!VMXNET3_VERSION_GE_6(adapter)) {
3975 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3977 if (VMXNET3_VERSION_GE_6(adapter)) {
3978 spin_lock_irqsave(&adapter->cmd_lock, flags);
3979 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3980 VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3981 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3982 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3984 rx_queues = (rx_queues >> 8) & 0xff;
3986 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3987 num_rx_queues = min(num_rx_queues, rx_queues);
3989 num_rx_queues = min(num_rx_queues,
3990 VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3993 cancel_work_sync(&adapter->work);
3995 unregister_netdev(netdev);
3997 vmxnet3_free_intr_resources(adapter);
3998 vmxnet3_free_pci_resources(adapter);
3999 if (VMXNET3_VERSION_GE_3(adapter)) {
4000 dma_free_coherent(&adapter->pdev->dev,
4001 sizeof(struct Vmxnet3_CoalesceScheme),
4002 adapter->coal_conf, adapter->coal_conf_pa);
4005 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
4006 adapter->rss_conf, adapter->rss_conf_pa);
4008 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
4009 adapter->pm_conf, adapter->pm_conf_pa);
4011 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
4012 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
4013 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
4014 adapter->queue_desc_pa);
4015 dma_free_coherent(&adapter->pdev->dev,
4016 sizeof(struct Vmxnet3_DriverShared),
4017 adapter->shared, adapter->shared_pa);
4018 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
4019 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
4020 free_netdev(netdev);
4023 static void vmxnet3_shutdown_device(struct pci_dev *pdev)
4025 struct net_device *netdev = pci_get_drvdata(pdev);
4026 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4027 unsigned long flags;
4029 /* Reset_work may be in the middle of resetting the device, wait for its
4032 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
4033 usleep_range(1000, 2000);
4035 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
4037 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4040 spin_lock_irqsave(&adapter->cmd_lock, flags);
4041 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4042 VMXNET3_CMD_QUIESCE_DEV);
4043 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4044 vmxnet3_disable_all_intrs(adapter);
4046 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4053 vmxnet3_suspend(struct device *device)
4055 struct pci_dev *pdev = to_pci_dev(device);
4056 struct net_device *netdev = pci_get_drvdata(pdev);
4057 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4058 struct Vmxnet3_PMConf *pmConf;
4059 struct ethhdr *ehdr;
4060 struct arphdr *ahdr;
4062 struct in_device *in_dev;
4063 struct in_ifaddr *ifa;
4064 unsigned long flags;
4067 if (!netif_running(netdev))
4070 for (i = 0; i < adapter->num_rx_queues; i++)
4071 napi_disable(&adapter->rx_queue[i].napi);
4073 vmxnet3_disable_all_intrs(adapter);
4074 vmxnet3_free_irqs(adapter);
4075 vmxnet3_free_intr_resources(adapter);
4077 netif_device_detach(netdev);
4079 /* Create wake-up filters. */
4080 pmConf = adapter->pm_conf;
4081 memset(pmConf, 0, sizeof(*pmConf));
4083 if (adapter->wol & WAKE_UCAST) {
4084 pmConf->filters[i].patternSize = ETH_ALEN;
4085 pmConf->filters[i].maskSize = 1;
4086 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
4087 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
4089 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4093 if (adapter->wol & WAKE_ARP) {
4096 in_dev = __in_dev_get_rcu(netdev);
4102 ifa = rcu_dereference(in_dev->ifa_list);
4108 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
4109 sizeof(struct arphdr) + /* ARP header */
4110 2 * ETH_ALEN + /* 2 Ethernet addresses*/
4111 2 * sizeof(u32); /*2 IPv4 addresses */
4112 pmConf->filters[i].maskSize =
4113 (pmConf->filters[i].patternSize - 1) / 8 + 1;
4115 /* ETH_P_ARP in Ethernet header. */
4116 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
4117 ehdr->h_proto = htons(ETH_P_ARP);
4119 /* ARPOP_REQUEST in ARP header. */
4120 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
4121 ahdr->ar_op = htons(ARPOP_REQUEST);
4122 arpreq = (u8 *)(ahdr + 1);
4124 /* The Unicast IPv4 address in 'tip' field. */
4125 arpreq += 2 * ETH_ALEN + sizeof(u32);
4126 *(__be32 *)arpreq = ifa->ifa_address;
4130 /* The mask for the relevant bits. */
4131 pmConf->filters[i].mask[0] = 0x00;
4132 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
4133 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
4134 pmConf->filters[i].mask[3] = 0x00;
4135 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
4136 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
4138 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4143 if (adapter->wol & WAKE_MAGIC)
4144 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
4146 pmConf->numFilters = i;
4148 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
4149 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
4151 adapter->shared->devRead.pmConfDesc.confPA =
4152 cpu_to_le64(adapter->pm_conf_pa);
4154 spin_lock_irqsave(&adapter->cmd_lock, flags);
4155 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4156 VMXNET3_CMD_UPDATE_PMCFG);
4157 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4159 pci_save_state(pdev);
4160 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
4162 pci_disable_device(pdev);
4163 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
4170 vmxnet3_resume(struct device *device)
4173 unsigned long flags;
4174 struct pci_dev *pdev = to_pci_dev(device);
4175 struct net_device *netdev = pci_get_drvdata(pdev);
4176 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4178 if (!netif_running(netdev))
4181 pci_set_power_state(pdev, PCI_D0);
4182 pci_restore_state(pdev);
4183 err = pci_enable_device_mem(pdev);
4187 pci_enable_wake(pdev, PCI_D0, 0);
4189 vmxnet3_alloc_intr_resources(adapter);
4191 /* During hibernate and suspend, device has to be reinitialized as the
4192 * device state need not be preserved.
4195 /* Need not check adapter state as other reset tasks cannot run during
4198 spin_lock_irqsave(&adapter->cmd_lock, flags);
4199 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4200 VMXNET3_CMD_QUIESCE_DEV);
4201 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4202 vmxnet3_tq_cleanup_all(adapter);
4203 vmxnet3_rq_cleanup_all(adapter);
4205 vmxnet3_reset_dev(adapter);
4206 err = vmxnet3_activate_dev(adapter);
4209 "failed to re-activate on resume, error: %d", err);
4210 vmxnet3_force_close(adapter);
4213 netif_device_attach(netdev);
4218 static const struct dev_pm_ops vmxnet3_pm_ops = {
4219 .suspend = vmxnet3_suspend,
4220 .resume = vmxnet3_resume,
4221 .freeze = vmxnet3_suspend,
4222 .restore = vmxnet3_resume,
4226 static struct pci_driver vmxnet3_driver = {
4227 .name = vmxnet3_driver_name,
4228 .id_table = vmxnet3_pciid_table,
4229 .probe = vmxnet3_probe_device,
4230 .remove = vmxnet3_remove_device,
4231 .shutdown = vmxnet3_shutdown_device,
4233 .driver.pm = &vmxnet3_pm_ops,
4239 vmxnet3_init_module(void)
4241 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
4242 VMXNET3_DRIVER_VERSION_REPORT);
4243 return pci_register_driver(&vmxnet3_driver);
4246 module_init(vmxnet3_init_module);
4250 vmxnet3_exit_module(void)
4252 pci_unregister_driver(&vmxnet3_driver);
4255 module_exit(vmxnet3_exit_module);
4257 MODULE_AUTHOR("VMware, Inc.");
4258 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
4259 MODULE_LICENSE("GPL v2");
4260 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);