1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
5 #include <linux/ipv6.h>
6 #include <linux/if_vlan.h>
7 #include <net/ip6_checksum.h>
10 #include "ionic_lif.h"
11 #include "ionic_txrx.h"
13 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
14 ionic_desc_cb cb_func, void *cb_arg)
16 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
19 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
20 ionic_desc_cb cb_func, void *cb_arg)
22 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
25 bool ionic_txq_poke_doorbell(struct ionic_queue *q)
27 unsigned long now, then, dif;
28 struct netdev_queue *netdev_txq;
29 struct net_device *netdev;
31 netdev = q->lif->netdev;
32 netdev_txq = netdev_get_tx_queue(netdev, q->index);
34 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
36 if (q->tail_idx == q->head_idx) {
37 HARD_TX_UNLOCK(netdev, netdev_txq);
41 now = READ_ONCE(jiffies);
42 then = q->dbell_jiffies;
45 if (dif > q->dbell_deadline) {
46 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
47 q->dbval | q->head_idx);
49 q->dbell_jiffies = now;
52 HARD_TX_UNLOCK(netdev, netdev_txq);
57 bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
59 unsigned long now, then, dif;
61 /* no lock, called from rx napi or txrx napi, nothing else can fill */
63 if (q->tail_idx == q->head_idx)
66 now = READ_ONCE(jiffies);
67 then = q->dbell_jiffies;
70 if (dif > q->dbell_deadline) {
71 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
72 q->dbval | q->head_idx);
74 q->dbell_jiffies = now;
76 dif = 2 * q->dbell_deadline;
77 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
78 dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
80 q->dbell_deadline = dif;
86 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
88 return netdev_get_tx_queue(q->lif->netdev, q->index);
91 static int ionic_rx_page_alloc(struct ionic_queue *q,
92 struct ionic_buf_info *buf_info)
94 struct net_device *netdev = q->lif->netdev;
95 struct ionic_rx_stats *stats;
100 stats = q_to_rx_stats(q);
102 if (unlikely(!buf_info)) {
103 net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
104 netdev->name, q->name);
108 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
109 if (unlikely(!page)) {
110 net_err_ratelimited("%s: %s page alloc failed\n",
111 netdev->name, q->name);
116 buf_info->dma_addr = dma_map_page(dev, page, 0,
117 IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
118 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
119 __free_pages(page, 0);
120 net_err_ratelimited("%s: %s dma map failed\n",
121 netdev->name, q->name);
122 stats->dma_map_err++;
126 buf_info->page = page;
127 buf_info->page_offset = 0;
132 static void ionic_rx_page_free(struct ionic_queue *q,
133 struct ionic_buf_info *buf_info)
135 struct net_device *netdev = q->lif->netdev;
136 struct device *dev = q->dev;
138 if (unlikely(!buf_info)) {
139 net_err_ratelimited("%s: %s invalid buf_info in free\n",
140 netdev->name, q->name);
147 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
148 __free_pages(buf_info->page, 0);
149 buf_info->page = NULL;
152 static bool ionic_rx_buf_recycle(struct ionic_queue *q,
153 struct ionic_buf_info *buf_info, u32 used)
157 /* don't re-use pages allocated in low-mem condition */
158 if (page_is_pfmemalloc(buf_info->page))
161 /* don't re-use buffers from non-local numa nodes */
162 if (page_to_nid(buf_info->page) != numa_mem_id())
165 size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
166 buf_info->page_offset += size;
167 if (buf_info->page_offset >= IONIC_PAGE_SIZE)
170 get_page(buf_info->page);
175 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
176 struct ionic_desc_info *desc_info,
177 struct ionic_rxq_comp *comp)
179 struct net_device *netdev = q->lif->netdev;
180 struct ionic_buf_info *buf_info;
181 struct ionic_rx_stats *stats;
182 struct device *dev = q->dev;
188 stats = q_to_rx_stats(q);
190 buf_info = &desc_info->bufs[0];
191 len = le16_to_cpu(comp->len);
193 prefetchw(buf_info->page);
195 skb = napi_get_frags(&q_to_qcq(q)->napi);
196 if (unlikely(!skb)) {
197 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
198 netdev->name, q->name);
203 i = comp->num_sg_elems + 1;
205 if (unlikely(!buf_info->page)) {
210 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset);
213 dma_sync_single_for_cpu(dev,
214 buf_info->dma_addr + buf_info->page_offset,
215 frag_len, DMA_FROM_DEVICE);
217 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
218 buf_info->page, buf_info->page_offset, frag_len,
221 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
222 dma_unmap_page(dev, buf_info->dma_addr,
223 IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
224 buf_info->page = NULL;
235 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
236 struct ionic_desc_info *desc_info,
237 struct ionic_rxq_comp *comp)
239 struct net_device *netdev = q->lif->netdev;
240 struct ionic_buf_info *buf_info;
241 struct ionic_rx_stats *stats;
242 struct device *dev = q->dev;
246 stats = q_to_rx_stats(q);
248 buf_info = &desc_info->bufs[0];
249 len = le16_to_cpu(comp->len);
251 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
252 if (unlikely(!skb)) {
253 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
254 netdev->name, q->name);
259 if (unlikely(!buf_info->page)) {
264 dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
265 len, DMA_FROM_DEVICE);
266 skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
267 dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
268 len, DMA_FROM_DEVICE);
271 skb->protocol = eth_type_trans(skb, q->lif->netdev);
276 static void ionic_rx_clean(struct ionic_queue *q,
277 struct ionic_desc_info *desc_info,
278 struct ionic_cq_info *cq_info,
281 struct net_device *netdev = q->lif->netdev;
282 struct ionic_qcq *qcq = q_to_qcq(q);
283 struct ionic_rx_stats *stats;
284 struct ionic_rxq_comp *comp;
287 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
289 stats = q_to_rx_stats(q);
297 stats->bytes += le16_to_cpu(comp->len);
299 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
300 skb = ionic_rx_copybreak(q, desc_info, comp);
302 skb = ionic_rx_frags(q, desc_info, comp);
304 if (unlikely(!skb)) {
309 skb_record_rx_queue(skb, q->index);
311 if (likely(netdev->features & NETIF_F_RXHASH)) {
312 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
313 case IONIC_PKT_TYPE_IPV4:
314 case IONIC_PKT_TYPE_IPV6:
315 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
318 case IONIC_PKT_TYPE_IPV4_TCP:
319 case IONIC_PKT_TYPE_IPV6_TCP:
320 case IONIC_PKT_TYPE_IPV4_UDP:
321 case IONIC_PKT_TYPE_IPV6_UDP:
322 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
328 if (likely(netdev->features & NETIF_F_RXCSUM) &&
329 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
330 skb->ip_summed = CHECKSUM_COMPLETE;
331 skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
332 stats->csum_complete++;
337 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
338 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
339 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
342 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
343 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
344 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
345 le16_to_cpu(comp->vlan_tci));
346 stats->vlan_stripped++;
349 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
350 __le64 *cq_desc_hwstamp;
356 sizeof(struct ionic_rxq_comp) -
357 IONIC_HWSTAMP_CQ_NEGOFFSET;
359 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
361 if (hwstamp != IONIC_HWSTAMP_INVALID) {
362 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
363 stats->hwstamp_valid++;
365 stats->hwstamp_invalid++;
369 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
370 napi_gro_receive(&qcq->napi, skb);
372 napi_gro_frags(&qcq->napi);
375 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
377 struct ionic_queue *q = cq->bound_q;
378 struct ionic_desc_info *desc_info;
379 struct ionic_rxq_comp *comp;
381 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
383 if (!color_match(comp->pkt_type_color, cq->done_color))
386 /* check for empty queue */
387 if (q->tail_idx == q->head_idx)
390 if (q->tail_idx != le16_to_cpu(comp->comp_index))
393 desc_info = &q->info[q->tail_idx];
394 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
396 /* clean the related q entry, only one per qc completion */
397 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
399 desc_info->cb = NULL;
400 desc_info->cb_arg = NULL;
405 static inline void ionic_write_cmb_desc(struct ionic_queue *q,
406 void __iomem *cmb_desc,
409 if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)
410 memcpy_toio(cmb_desc, desc, q->desc_size);
413 void ionic_rx_fill(struct ionic_queue *q)
415 struct net_device *netdev = q->lif->netdev;
416 struct ionic_desc_info *desc_info;
417 struct ionic_rxq_sg_desc *sg_desc;
418 struct ionic_rxq_sg_elem *sg_elem;
419 struct ionic_buf_info *buf_info;
420 unsigned int fill_threshold;
421 struct ionic_rxq_desc *desc;
422 unsigned int remain_len;
423 unsigned int frag_len;
429 n_fill = ionic_q_space_avail(q);
431 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
432 q->num_descs / IONIC_RX_FILL_DIV);
433 if (n_fill < fill_threshold)
436 len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
438 for (i = n_fill; i; i--) {
441 desc_info = &q->info[q->head_idx];
442 desc = desc_info->desc;
443 buf_info = &desc_info->bufs[0];
445 if (!buf_info->page) { /* alloc a new buffer? */
446 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
453 /* fill main descriptor - buf[0] */
454 desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
455 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset);
456 desc->len = cpu_to_le16(frag_len);
457 remain_len -= frag_len;
461 /* fill sg descriptors - buf[1..n] */
462 sg_desc = desc_info->sg_desc;
463 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
464 sg_elem = &sg_desc->elems[j];
465 if (!buf_info->page) { /* alloc a new sg buffer? */
466 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
473 sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
474 frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset);
475 sg_elem->len = cpu_to_le16(frag_len);
476 remain_len -= frag_len;
481 /* clear end sg element as a sentinel */
482 if (j < q->max_sg_elems) {
483 sg_elem = &sg_desc->elems[j];
484 memset(sg_elem, 0, sizeof(*sg_elem));
487 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
488 IONIC_RXQ_DESC_OPCODE_SIMPLE;
489 desc_info->nbufs = nfrags;
491 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
493 ionic_rxq_post(q, false, ionic_rx_clean, NULL);
496 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
497 q->dbval | q->head_idx);
499 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
500 q->dbell_jiffies = jiffies;
502 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
503 jiffies + IONIC_NAPI_DEADLINE);
506 void ionic_rx_empty(struct ionic_queue *q)
508 struct ionic_desc_info *desc_info;
509 struct ionic_buf_info *buf_info;
512 for (i = 0; i < q->num_descs; i++) {
513 desc_info = &q->info[i];
514 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
515 buf_info = &desc_info->bufs[j];
517 ionic_rx_page_free(q, buf_info);
520 desc_info->nbufs = 0;
521 desc_info->cb = NULL;
522 desc_info->cb_arg = NULL;
529 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
531 struct dim_sample dim_sample;
532 struct ionic_lif *lif;
536 if (!qcq->intr.dim_coal_hw)
540 qi = qcq->cq.bound_q->index;
543 case IONIC_LIF_F_TX_DIM_INTR:
544 pkts = lif->txqstats[qi].pkts;
545 bytes = lif->txqstats[qi].bytes;
547 case IONIC_LIF_F_RX_DIM_INTR:
548 pkts = lif->rxqstats[qi].pkts;
549 bytes = lif->rxqstats[qi].bytes;
552 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
553 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
557 dim_update_sample(qcq->cq.bound_intr->rearm_count,
558 pkts, bytes, &dim_sample);
560 net_dim(&qcq->dim, dim_sample);
563 int ionic_tx_napi(struct napi_struct *napi, int budget)
565 struct ionic_qcq *qcq = napi_to_qcq(napi);
566 struct ionic_cq *cq = napi_to_cq(napi);
567 struct ionic_dev *idev;
568 struct ionic_lif *lif;
572 lif = cq->bound_q->lif;
573 idev = &lif->ionic->idev;
575 work_done = ionic_cq_service(cq, budget,
576 ionic_tx_service, NULL, NULL);
578 if (work_done < budget && napi_complete_done(napi, work_done)) {
579 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
580 flags |= IONIC_INTR_CRED_UNMASK;
581 cq->bound_intr->rearm_count++;
584 if (work_done || flags) {
585 flags |= IONIC_INTR_CRED_RESET_COALESCE;
586 ionic_intr_credits(idev->intr_ctrl,
587 cq->bound_intr->index,
591 if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
592 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
597 int ionic_rx_napi(struct napi_struct *napi, int budget)
599 struct ionic_qcq *qcq = napi_to_qcq(napi);
600 struct ionic_cq *cq = napi_to_cq(napi);
601 struct ionic_dev *idev;
602 struct ionic_lif *lif;
606 lif = cq->bound_q->lif;
607 idev = &lif->ionic->idev;
609 work_done = ionic_cq_service(cq, budget,
610 ionic_rx_service, NULL, NULL);
612 ionic_rx_fill(cq->bound_q);
614 if (work_done < budget && napi_complete_done(napi, work_done)) {
615 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
616 flags |= IONIC_INTR_CRED_UNMASK;
617 cq->bound_intr->rearm_count++;
620 if (work_done || flags) {
621 flags |= IONIC_INTR_CRED_RESET_COALESCE;
622 ionic_intr_credits(idev->intr_ctrl,
623 cq->bound_intr->index,
627 if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
628 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
633 int ionic_txrx_napi(struct napi_struct *napi, int budget)
635 struct ionic_qcq *rxqcq = napi_to_qcq(napi);
636 struct ionic_cq *rxcq = napi_to_cq(napi);
637 unsigned int qi = rxcq->bound_q->index;
638 struct ionic_qcq *txqcq;
639 struct ionic_dev *idev;
640 struct ionic_lif *lif;
641 struct ionic_cq *txcq;
642 bool resched = false;
643 u32 rx_work_done = 0;
644 u32 tx_work_done = 0;
647 lif = rxcq->bound_q->lif;
648 idev = &lif->ionic->idev;
649 txqcq = lif->txqcqs[qi];
650 txcq = &lif->txqcqs[qi]->cq;
652 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT,
653 ionic_tx_service, NULL, NULL);
655 rx_work_done = ionic_cq_service(rxcq, budget,
656 ionic_rx_service, NULL, NULL);
658 ionic_rx_fill(rxcq->bound_q);
660 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
661 ionic_dim_update(rxqcq, 0);
662 flags |= IONIC_INTR_CRED_UNMASK;
663 rxcq->bound_intr->rearm_count++;
666 if (rx_work_done || flags) {
667 flags |= IONIC_INTR_CRED_RESET_COALESCE;
668 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
669 tx_work_done + rx_work_done, flags);
672 if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
674 if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
677 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
682 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
683 void *data, size_t len)
685 struct ionic_tx_stats *stats = q_to_tx_stats(q);
686 struct device *dev = q->dev;
689 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
690 if (dma_mapping_error(dev, dma_addr)) {
691 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
692 q->lif->netdev->name, q->name);
693 stats->dma_map_err++;
699 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
700 const skb_frag_t *frag,
701 size_t offset, size_t len)
703 struct ionic_tx_stats *stats = q_to_tx_stats(q);
704 struct device *dev = q->dev;
707 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
708 if (dma_mapping_error(dev, dma_addr)) {
709 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
710 q->lif->netdev->name, q->name);
711 stats->dma_map_err++;
716 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
717 struct ionic_desc_info *desc_info)
719 struct ionic_buf_info *buf_info = desc_info->bufs;
720 struct ionic_tx_stats *stats = q_to_tx_stats(q);
721 struct device *dev = q->dev;
727 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
728 if (dma_mapping_error(dev, dma_addr)) {
729 stats->dma_map_err++;
732 buf_info->dma_addr = dma_addr;
733 buf_info->len = skb_headlen(skb);
736 frag = skb_shinfo(skb)->frags;
737 nfrags = skb_shinfo(skb)->nr_frags;
738 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
739 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
740 if (dma_mapping_error(dev, dma_addr)) {
741 stats->dma_map_err++;
744 buf_info->dma_addr = dma_addr;
745 buf_info->len = skb_frag_size(frag);
749 desc_info->nbufs = 1 + nfrags;
754 /* unwind the frag mappings and the head mapping */
755 while (frag_idx > 0) {
758 dma_unmap_page(dev, buf_info->dma_addr,
759 buf_info->len, DMA_TO_DEVICE);
761 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
765 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
766 struct ionic_desc_info *desc_info)
768 struct ionic_buf_info *buf_info = desc_info->bufs;
769 struct device *dev = q->dev;
772 if (!desc_info->nbufs)
775 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
776 buf_info->len, DMA_TO_DEVICE);
778 for (i = 1; i < desc_info->nbufs; i++, buf_info++)
779 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
780 buf_info->len, DMA_TO_DEVICE);
782 desc_info->nbufs = 0;
785 static void ionic_tx_clean(struct ionic_queue *q,
786 struct ionic_desc_info *desc_info,
787 struct ionic_cq_info *cq_info,
790 struct ionic_tx_stats *stats = q_to_tx_stats(q);
791 struct ionic_qcq *qcq = q_to_qcq(q);
792 struct sk_buff *skb = cb_arg;
795 ionic_tx_desc_unmap_bufs(q, desc_info);
800 qi = skb_get_queue_mapping(skb);
802 if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) {
804 struct skb_shared_hwtstamps hwts = {};
805 __le64 *cq_desc_hwstamp;
811 sizeof(struct ionic_txq_comp) -
812 IONIC_HWSTAMP_CQ_NEGOFFSET;
814 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
816 if (hwstamp != IONIC_HWSTAMP_INVALID) {
817 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
819 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
820 skb_tstamp_tx(skb, &hwts);
822 stats->hwstamp_valid++;
824 stats->hwstamp_invalid++;
828 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {
829 netif_wake_subqueue(q->lif->netdev, qi);
832 desc_info->bytes = skb->len;
835 dev_consume_skb_any(skb);
838 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
840 struct ionic_queue *q = cq->bound_q;
841 struct ionic_desc_info *desc_info;
842 struct ionic_txq_comp *comp;
847 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
849 if (!color_match(comp->color, cq->done_color))
852 /* clean the related q entries, there could be
853 * several q entries completed for each cq completion
856 desc_info = &q->info[q->tail_idx];
857 desc_info->bytes = 0;
859 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
860 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
861 if (desc_info->cb_arg) {
863 bytes += desc_info->bytes;
865 desc_info->cb = NULL;
866 desc_info->cb_arg = NULL;
867 } while (index != le16_to_cpu(comp->comp_index));
869 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
870 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
875 void ionic_tx_flush(struct ionic_cq *cq)
877 struct ionic_dev *idev = &cq->lif->ionic->idev;
880 work_done = ionic_cq_service(cq, cq->num_descs,
881 ionic_tx_service, NULL, NULL);
883 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
884 work_done, IONIC_INTR_CRED_RESET_COALESCE);
887 void ionic_tx_empty(struct ionic_queue *q)
889 struct ionic_desc_info *desc_info;
893 /* walk the not completed tx entries, if any */
894 while (q->head_idx != q->tail_idx) {
895 desc_info = &q->info[q->tail_idx];
896 desc_info->bytes = 0;
897 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
898 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
899 if (desc_info->cb_arg) {
901 bytes += desc_info->bytes;
903 desc_info->cb = NULL;
904 desc_info->cb_arg = NULL;
907 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
908 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
911 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
915 err = skb_cow_head(skb, 0);
919 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
920 inner_ip_hdr(skb)->check = 0;
921 inner_tcp_hdr(skb)->check =
922 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
923 inner_ip_hdr(skb)->daddr,
925 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
926 inner_tcp_hdr(skb)->check =
927 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
928 &inner_ipv6_hdr(skb)->daddr,
935 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
939 err = skb_cow_head(skb, 0);
943 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
944 ip_hdr(skb)->check = 0;
945 tcp_hdr(skb)->check =
946 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
949 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
950 tcp_v6_gso_csum_prep(skb);
956 static void ionic_tx_tso_post(struct ionic_queue *q,
957 struct ionic_desc_info *desc_info,
959 dma_addr_t addr, u8 nsge, u16 len,
960 unsigned int hdrlen, unsigned int mss,
962 u16 vlan_tci, bool has_vlan,
963 bool start, bool done)
965 struct ionic_txq_desc *desc = desc_info->desc;
969 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
970 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
971 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
972 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
974 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
975 desc->cmd = cpu_to_le64(cmd);
976 desc->len = cpu_to_le16(len);
977 desc->vlan_tci = cpu_to_le16(vlan_tci);
978 desc->hdr_len = cpu_to_le16(hdrlen);
979 desc->mss = cpu_to_le16(mss);
981 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
984 skb_tx_timestamp(skb);
985 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
986 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
987 ionic_txq_post(q, false, ionic_tx_clean, skb);
989 ionic_txq_post(q, done, NULL, NULL);
993 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
995 struct ionic_tx_stats *stats = q_to_tx_stats(q);
996 struct ionic_desc_info *desc_info;
997 struct ionic_buf_info *buf_info;
998 struct ionic_txq_sg_elem *elem;
999 struct ionic_txq_desc *desc;
1000 unsigned int chunk_len;
1001 unsigned int frag_rem;
1002 unsigned int tso_rem;
1003 unsigned int seg_rem;
1004 dma_addr_t desc_addr;
1005 dma_addr_t frag_addr;
1006 unsigned int hdrlen;
1018 desc_info = &q->info[q->head_idx];
1019 buf_info = desc_info->bufs;
1021 if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1025 mss = skb_shinfo(skb)->gso_size;
1026 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1030 SKB_GSO_UDP_TUNNEL |
1031 SKB_GSO_UDP_TUNNEL_CSUM));
1032 has_vlan = !!skb_vlan_tag_present(skb);
1033 vlan_tci = skb_vlan_tag_get(skb);
1034 encap = skb->encapsulation;
1036 /* Preload inner-most TCP csum field with IP pseudo hdr
1037 * calculated with IP length set to zero. HW will later
1038 * add in length to each TCP segment resulting from the TSO.
1042 err = ionic_tx_tcp_inner_pseudo_csum(skb);
1044 err = ionic_tx_tcp_pseudo_csum(skb);
1046 /* clean up mapping from ionic_tx_map_skb */
1047 ionic_tx_desc_unmap_bufs(q, desc_info);
1052 hdrlen = skb_inner_tcp_all_headers(skb);
1054 hdrlen = skb_tcp_all_headers(skb);
1057 seg_rem = min(tso_rem, hdrlen + mss);
1064 while (tso_rem > 0) {
1070 /* use fragments until we have enough to post a single descriptor */
1071 while (seg_rem > 0) {
1072 /* if the fragment is exhausted then move to the next one */
1073 if (frag_rem == 0) {
1074 /* grab the next fragment */
1075 frag_addr = buf_info->dma_addr;
1076 frag_rem = buf_info->len;
1079 chunk_len = min(frag_rem, seg_rem);
1081 /* fill main descriptor */
1082 desc = desc_info->txq_desc;
1083 elem = desc_info->txq_sg_desc->elems;
1084 desc_addr = frag_addr;
1085 desc_len = chunk_len;
1087 /* fill sg descriptor */
1088 elem->addr = cpu_to_le64(frag_addr);
1089 elem->len = cpu_to_le16(chunk_len);
1093 frag_addr += chunk_len;
1094 frag_rem -= chunk_len;
1095 tso_rem -= chunk_len;
1096 seg_rem -= chunk_len;
1098 seg_rem = min(tso_rem, mss);
1099 done = (tso_rem == 0);
1100 /* post descriptor */
1101 ionic_tx_tso_post(q, desc_info, skb,
1102 desc_addr, desc_nsge, desc_len,
1103 hdrlen, mss, outer_csum, vlan_tci, has_vlan,
1106 /* Buffer information is stored with the first tso descriptor */
1107 desc_info = &q->info[q->head_idx];
1108 desc_info->nbufs = 0;
1111 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
1112 stats->bytes += len;
1114 stats->tso_bytes = len;
1119 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
1120 struct ionic_desc_info *desc_info)
1122 struct ionic_txq_desc *desc = desc_info->txq_desc;
1123 struct ionic_buf_info *buf_info = desc_info->bufs;
1124 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1130 has_vlan = !!skb_vlan_tag_present(skb);
1131 encap = skb->encapsulation;
1133 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1134 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1136 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
1137 flags, skb_shinfo(skb)->nr_frags,
1138 buf_info->dma_addr);
1139 desc->cmd = cpu_to_le64(cmd);
1140 desc->len = cpu_to_le16(buf_info->len);
1142 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1143 stats->vlan_inserted++;
1147 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
1148 desc->csum_offset = cpu_to_le16(skb->csum_offset);
1150 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
1152 if (skb_csum_is_sctp(skb))
1153 stats->crc32_csum++;
1158 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
1159 struct ionic_desc_info *desc_info)
1161 struct ionic_txq_desc *desc = desc_info->txq_desc;
1162 struct ionic_buf_info *buf_info = desc_info->bufs;
1163 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1169 has_vlan = !!skb_vlan_tag_present(skb);
1170 encap = skb->encapsulation;
1172 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1173 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1175 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
1176 flags, skb_shinfo(skb)->nr_frags,
1177 buf_info->dma_addr);
1178 desc->cmd = cpu_to_le64(cmd);
1179 desc->len = cpu_to_le16(buf_info->len);
1181 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1182 stats->vlan_inserted++;
1186 desc->csum_start = 0;
1187 desc->csum_offset = 0;
1189 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
1194 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
1195 struct ionic_desc_info *desc_info)
1197 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
1198 struct ionic_buf_info *buf_info = &desc_info->bufs[1];
1199 struct ionic_txq_sg_elem *elem = sg_desc->elems;
1200 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1203 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
1204 elem->addr = cpu_to_le64(buf_info->dma_addr);
1205 elem->len = cpu_to_le16(buf_info->len);
1208 stats->frags += skb_shinfo(skb)->nr_frags;
1211 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
1213 struct ionic_desc_info *desc_info = &q->info[q->head_idx];
1214 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1216 if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1219 /* set up the initial descriptor */
1220 if (skb->ip_summed == CHECKSUM_PARTIAL)
1221 ionic_tx_calc_csum(q, skb, desc_info);
1223 ionic_tx_calc_no_csum(q, skb, desc_info);
1226 ionic_tx_skb_frags(q, skb, desc_info);
1228 skb_tx_timestamp(skb);
1230 stats->bytes += skb->len;
1232 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
1233 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
1234 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
1239 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1241 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1245 /* Each desc is mss long max, so a descriptor for each gso_seg */
1246 if (skb_is_gso(skb))
1247 ndescs = skb_shinfo(skb)->gso_segs;
1251 /* If non-TSO, just need 1 desc and nr_frags sg elems */
1252 if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems)
1255 /* Too many frags, so linearize */
1256 err = skb_linearize(skb);
1265 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1269 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1270 netif_stop_subqueue(q->lif->netdev, q->index);
1273 /* Might race with ionic_tx_clean, check again */
1275 if (ionic_q_has_space(q, ndescs)) {
1276 netif_wake_subqueue(q->lif->netdev, q->index);
1284 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1285 struct net_device *netdev)
1287 struct ionic_lif *lif = netdev_priv(netdev);
1288 struct ionic_queue *q = &lif->hwstamp_txq->q;
1291 /* Does not stop/start txq, because we post to a separate tx queue
1292 * for timestamping, and if a packet can't be posted immediately to
1293 * the timestamping queue, it is dropped.
1296 ndescs = ionic_tx_descs_needed(q, skb);
1297 if (unlikely(ndescs < 0))
1300 if (unlikely(!ionic_q_has_space(q, ndescs)))
1303 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1304 if (skb_is_gso(skb))
1305 err = ionic_tx_tso(q, skb);
1307 err = ionic_tx(q, skb);
1312 return NETDEV_TX_OK;
1317 return NETDEV_TX_OK;
1320 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1322 u16 queue_index = skb_get_queue_mapping(skb);
1323 struct ionic_lif *lif = netdev_priv(netdev);
1324 struct ionic_queue *q;
1328 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1330 return NETDEV_TX_OK;
1333 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1334 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1335 return ionic_start_hwstamp_xmit(skb, netdev);
1337 if (unlikely(queue_index >= lif->nxqs))
1339 q = &lif->txqcqs[queue_index]->q;
1341 ndescs = ionic_tx_descs_needed(q, skb);
1345 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1346 return NETDEV_TX_BUSY;
1348 if (skb_is_gso(skb))
1349 err = ionic_tx_tso(q, skb);
1351 err = ionic_tx(q, skb);
1356 /* Stop the queue if there aren't descriptors for the next packet.
1357 * Since our SG lists per descriptor take care of most of the possible
1358 * fragmentation, we don't need to have many descriptors available.
1360 ionic_maybe_stop_tx(q, 4);
1362 return NETDEV_TX_OK;
1367 return NETDEV_TX_OK;