drivers/net/ethernet/marvell/octeon_ep/octep_tx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
   3  *
   4  * Copyright (C) 2020 Marvell.
   5  *
   6  */
   7
   8 #include <linux/pci.h>
   9 #include <linux/etherdevice.h>
  10 #include <linux/vmalloc.h>
  11
  12 #include "octep_config.h"
  13 #include "octep_main.h"
  14
  15 /* Reset various index of Tx queue data structure. */
  16 static void octep_iq_reset_indices(struct octep_iq *iq)
  17 {
  18         iq->fill_cnt = 0;
  19         iq->host_write_index = 0;
  20         iq->octep_read_index = 0;
  21         iq->flush_index = 0;
  22         iq->pkts_processed = 0;
  23         iq->pkt_in_done = 0;
  24         atomic_set(&iq->instr_pending, 0);
  25 }
  26
  27 /**
  28  * octep_iq_process_completions() - Process Tx queue completions.
  29  *
  30  * @iq: Octeon Tx queue data structure.
  31  * @budget: max number of completions to be processed in one invocation.
  32  */
  33 int octep_iq_process_completions(struct octep_iq *iq, u16 budget)
  34 {
  35         u32 compl_pkts, compl_bytes, compl_sg;
  36         struct octep_device *oct = iq->octep_dev;
  37         struct octep_tx_buffer *tx_buffer;
  38         struct skb_shared_info *shinfo;
  39         u32 fi = iq->flush_index;
  40         struct sk_buff *skb;
  41         u8 frags, i;
  42
  43         compl_pkts = 0;
  44         compl_sg = 0;
  45         compl_bytes = 0;
  46         iq->octep_read_index = oct->hw_ops.update_iq_read_idx(iq);
  47
  48         while (likely(budget && (fi != iq->octep_read_index))) {
  49                 tx_buffer = iq->buff_info + fi;
  50                 skb = tx_buffer->skb;
  51
  52                 fi++;
  53                 if (unlikely(fi == iq->max_count))
  54                         fi = 0;
  55                 compl_bytes += skb->len;
  56                 compl_pkts++;
  57                 budget--;
  58
  59                 if (!tx_buffer->gather) {
  60                         dma_unmap_single(iq->dev, tx_buffer->dma,
  61                                          tx_buffer->skb->len, DMA_TO_DEVICE);
  62                         dev_kfree_skb_any(skb);
  63                         continue;
  64                 }
  65
  66                 /* Scatter/Gather */
  67                 shinfo = skb_shinfo(skb);
  68                 frags = shinfo->nr_frags;
  69                 compl_sg++;
  70
  71                 dma_unmap_single(iq->dev, tx_buffer->sglist[0].dma_ptr[0],
  72                                  tx_buffer->sglist[0].len[3], DMA_TO_DEVICE);
  73
  74                 i = 1; /* entry 0 is main skb, unmapped above */
  75                 while (frags--) {
  76                         dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3],
  77                                        tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE);
  78                         i++;
  79                 }
  80
  81                 dev_kfree_skb_any(skb);
  82         }
  83
  84         iq->pkts_processed += compl_pkts;
  85         atomic_sub(compl_pkts, &iq->instr_pending);
  86         iq->stats.instr_completed += compl_pkts;
  87         iq->stats.bytes_sent += compl_bytes;
  88         iq->stats.sgentry_sent += compl_sg;
  89         iq->flush_index = fi;
  90
  91         netdev_tx_completed_queue(iq->netdev_q, compl_pkts, compl_bytes);
  92
  93         if (unlikely(__netif_subqueue_stopped(iq->netdev, iq->q_no)) &&
  94             ((iq->max_count - atomic_read(&iq->instr_pending)) >
  95              OCTEP_WAKE_QUEUE_THRESHOLD))
  96                 netif_wake_subqueue(iq->netdev, iq->q_no);
  97         return !budget;
  98 }
  99
 100 /**
 101  * octep_iq_free_pending() - Free Tx buffers for pending completions.
 102  *
 103  * @iq: Octeon Tx queue data structure.
 104  */
 105 static void octep_iq_free_pending(struct octep_iq *iq)
 106 {
 107         struct octep_tx_buffer *tx_buffer;
 108         struct skb_shared_info *shinfo;
 109         u32 fi = iq->flush_index;
 110         struct sk_buff *skb;
 111         u8 frags, i;
 112
 113         while (fi != iq->host_write_index) {
 114                 tx_buffer = iq->buff_info + fi;
 115                 skb = tx_buffer->skb;
 116
 117                 fi++;
 118                 if (unlikely(fi == iq->max_count))
 119                         fi = 0;
 120
 121                 if (!tx_buffer->gather) {
 122                         dma_unmap_single(iq->dev, tx_buffer->dma,
 123                                          tx_buffer->skb->len, DMA_TO_DEVICE);
 124                         dev_kfree_skb_any(skb);
 125                         continue;
 126                 }
 127
 128                 /* Scatter/Gather */
 129                 shinfo = skb_shinfo(skb);
 130                 frags = shinfo->nr_frags;
 131
 132                 dma_unmap_single(iq->dev,
 133                                  tx_buffer->sglist[0].dma_ptr[0],
 134                                  tx_buffer->sglist[0].len[3],
 135                                  DMA_TO_DEVICE);
 136
 137                 i = 1; /* entry 0 is main skb, unmapped above */
 138                 while (frags--) {
 139                         dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3],
 140                                        tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE);
 141                         i++;
 142                 }
 143
 144                 dev_kfree_skb_any(skb);
 145         }
 146
 147         atomic_set(&iq->instr_pending, 0);
 148         iq->flush_index = fi;
 149         netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no));
 150 }
 151
 152 /**
 153  * octep_clean_iqs()  - Clean Tx queues to shutdown the device.
 154  *
 155  * @oct: Octeon device private data structure.
 156  *
 157  * Free the buffers in Tx queue descriptors pending completion and
 158  * reset queue indices
 159  */
 160 void octep_clean_iqs(struct octep_device *oct)
 161 {
 162         int i;
 163
 164         for (i = 0; i < oct->num_iqs; i++) {
 165                 octep_iq_free_pending(oct->iq[i]);
 166                 octep_iq_reset_indices(oct->iq[i]);
 167         }
 168 }
 169
 170 /**
 171  * octep_setup_iq() - Setup a Tx queue.
 172  *
 173  * @oct: Octeon device private data structure.
 174  * @q_no: Tx queue number to be setup.
 175  *
 176  * Allocate resources for a Tx queue.
 177  */
 178 static int octep_setup_iq(struct octep_device *oct, int q_no)
 179 {
 180         u32 desc_ring_size, buff_info_size, sglist_size;
 181         struct octep_iq *iq;
 182         int i;
 183
 184         iq = vzalloc(sizeof(*iq));
 185         if (!iq)
 186                 goto iq_alloc_err;
 187         oct->iq[q_no] = iq;
 188
 189         iq->octep_dev = oct;
 190         iq->netdev = oct->netdev;
 191         iq->dev = &oct->pdev->dev;
 192         iq->q_no = q_no;
 193         iq->max_count = CFG_GET_IQ_NUM_DESC(oct->conf);
 194         iq->ring_size_mask = iq->max_count - 1;
 195         iq->fill_threshold = CFG_GET_IQ_DB_MIN(oct->conf);
 196         iq->netdev_q = netdev_get_tx_queue(iq->netdev, q_no);
 197
 198         /* Allocate memory for hardware queue descriptors */
 199         desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf);
 200         iq->desc_ring = dma_alloc_coherent(iq->dev, desc_ring_size,
 201                                            &iq->desc_ring_dma, GFP_KERNEL);
 202         if (unlikely(!iq->desc_ring)) {
 203                 dev_err(iq->dev,
 204                         "Failed to allocate DMA memory for IQ-%d\n", q_no);
 205                 goto desc_dma_alloc_err;
 206         }
 207
 208         /* Allocate memory for hardware SGLIST descriptors */
 209         sglist_size = OCTEP_SGLIST_SIZE_PER_PKT *
 210                       CFG_GET_IQ_NUM_DESC(oct->conf);
 211         iq->sglist = dma_alloc_coherent(iq->dev, sglist_size,
 212                                         &iq->sglist_dma, GFP_KERNEL);
 213         if (unlikely(!iq->sglist)) {
 214                 dev_err(iq->dev,
 215                         "Failed to allocate DMA memory for IQ-%d SGLIST\n",
 216                         q_no);
 217                 goto sglist_alloc_err;
 218         }
 219
 220         /* allocate memory to manage Tx packets pending completion */
 221         buff_info_size = OCTEP_IQ_TXBUFF_INFO_SIZE * iq->max_count;
 222         iq->buff_info = vzalloc(buff_info_size);
 223         if (!iq->buff_info) {
 224                 dev_err(iq->dev,
 225                         "Failed to allocate buff info for IQ-%d\n", q_no);
 226                 goto buff_info_err;
 227         }
 228
 229         /* Setup sglist addresses in tx_buffer entries */
 230         for (i = 0; i < CFG_GET_IQ_NUM_DESC(oct->conf); i++) {
 231                 struct octep_tx_buffer *tx_buffer;
 232
 233                 tx_buffer = &iq->buff_info[i];
 234                 tx_buffer->sglist =
 235                         &iq->sglist[i * OCTEP_SGLIST_ENTRIES_PER_PKT];
 236                 tx_buffer->sglist_dma =
 237                         iq->sglist_dma + (i * OCTEP_SGLIST_SIZE_PER_PKT);
 238         }
 239
 240         octep_iq_reset_indices(iq);
 241         oct->hw_ops.setup_iq_regs(oct, q_no);
 242
 243         oct->num_iqs++;
 244         return 0;
 245
 246 buff_info_err:
 247         dma_free_coherent(iq->dev, sglist_size, iq->sglist, iq->sglist_dma);
 248 sglist_alloc_err:
 249         dma_free_coherent(iq->dev, desc_ring_size,
 250                           iq->desc_ring, iq->desc_ring_dma);
 251 desc_dma_alloc_err:
 252         vfree(iq);
 253         oct->iq[q_no] = NULL;
 254 iq_alloc_err:
 255         return -1;
 256 }
 257
 258 /**
 259  * octep_free_iq() - Free Tx queue resources.
 260  *
 261  * @iq: Octeon Tx queue data structure.
 262  *
 263  * Free all the resources allocated for a Tx queue.
 264  */
 265 static void octep_free_iq(struct octep_iq *iq)
 266 {
 267         struct octep_device *oct = iq->octep_dev;
 268         u64 desc_ring_size, sglist_size;
 269         int q_no = iq->q_no;
 270
 271         desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf);
 272
 273         vfree(iq->buff_info);
 274
 275         if (iq->desc_ring)
 276                 dma_free_coherent(iq->dev, desc_ring_size,
 277                                   iq->desc_ring, iq->desc_ring_dma);
 278
 279         sglist_size = OCTEP_SGLIST_SIZE_PER_PKT *
 280                       CFG_GET_IQ_NUM_DESC(oct->conf);
 281         if (iq->sglist)
 282                 dma_free_coherent(iq->dev, sglist_size,
 283                                   iq->sglist, iq->sglist_dma);
 284
 285         vfree(iq);
 286         oct->iq[q_no] = NULL;
 287         oct->num_iqs--;
 288 }
 289
 290 /**
 291  * octep_setup_iqs() - setup resources for all Tx queues.
 292  *
 293  * @oct: Octeon device private data structure.
 294  */
 295 int octep_setup_iqs(struct octep_device *oct)
 296 {
 297         int i;
 298
 299         oct->num_iqs = 0;
 300         for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
 301                 if (octep_setup_iq(oct, i)) {
 302                         dev_err(&oct->pdev->dev,
 303                                 "Failed to setup IQ(TxQ)-%d.\n", i);
 304                         goto iq_setup_err;
 305                 }
 306                 dev_dbg(&oct->pdev->dev, "Successfully setup IQ(TxQ)-%d.\n", i);
 307         }
 308
 309         return 0;
 310
 311 iq_setup_err:
 312         while (i) {
 313                 i--;
 314                 octep_free_iq(oct->iq[i]);
 315         }
 316         return -1;
 317 }
 318
 319 /**
 320  * octep_free_iqs() - Free resources of all Tx queues.
 321  *
 322  * @oct: Octeon device private data structure.
 323  */
 324 void octep_free_iqs(struct octep_device *oct)
 325 {
 326         int i;
 327
 328         for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
 329                 octep_free_iq(oct->iq[i]);
 330                 dev_dbg(&oct->pdev->dev,
 331                         "Successfully destroyed IQ(TxQ)-%d.\n", i);
 332         }
 333         oct->num_iqs = 0;
 334 }