1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
3 * Microsemi SoCs FDMA driver
5 * Copyright (c) 2021 Microchip
7 * Page recycling code is mostly taken from gianfar driver.
10 #include <linux/align.h>
11 #include <linux/bitops.h>
12 #include <linux/dmapool.h>
13 #include <linux/dsa/ocelot.h>
14 #include <linux/netdevice.h>
15 #include <linux/of_platform.h>
16 #include <linux/skbuff.h>
18 #include "ocelot_fdma.h"
19 #include "ocelot_qs.h"
21 DEFINE_STATIC_KEY_FALSE(ocelot_fdma_enabled);
23 static void ocelot_fdma_writel(struct ocelot *ocelot, u32 reg, u32 data)
25 regmap_write(ocelot->targets[FDMA], reg, data);
28 static u32 ocelot_fdma_readl(struct ocelot *ocelot, u32 reg)
32 regmap_read(ocelot->targets[FDMA], reg, &retval);
37 static dma_addr_t ocelot_fdma_idx_dma(dma_addr_t base, u16 idx)
39 return base + idx * sizeof(struct ocelot_fdma_dcb);
42 static u16 ocelot_fdma_dma_idx(dma_addr_t base, dma_addr_t dma)
44 return (dma - base) / sizeof(struct ocelot_fdma_dcb);
47 static u16 ocelot_fdma_idx_next(u16 idx, u16 ring_sz)
49 return unlikely(idx == ring_sz - 1) ? 0 : idx + 1;
52 static u16 ocelot_fdma_idx_prev(u16 idx, u16 ring_sz)
54 return unlikely(idx == 0) ? ring_sz - 1 : idx - 1;
57 static int ocelot_fdma_rx_ring_free(struct ocelot_fdma *fdma)
59 struct ocelot_fdma_rx_ring *rx_ring = &fdma->rx_ring;
61 if (rx_ring->next_to_use >= rx_ring->next_to_clean)
62 return OCELOT_FDMA_RX_RING_SIZE -
63 (rx_ring->next_to_use - rx_ring->next_to_clean) - 1;
65 return rx_ring->next_to_clean - rx_ring->next_to_use - 1;
68 static int ocelot_fdma_tx_ring_free(struct ocelot_fdma *fdma)
70 struct ocelot_fdma_tx_ring *tx_ring = &fdma->tx_ring;
72 if (tx_ring->next_to_use >= tx_ring->next_to_clean)
73 return OCELOT_FDMA_TX_RING_SIZE -
74 (tx_ring->next_to_use - tx_ring->next_to_clean) - 1;
76 return tx_ring->next_to_clean - tx_ring->next_to_use - 1;
79 static bool ocelot_fdma_tx_ring_empty(struct ocelot_fdma *fdma)
81 struct ocelot_fdma_tx_ring *tx_ring = &fdma->tx_ring;
83 return tx_ring->next_to_clean == tx_ring->next_to_use;
86 static void ocelot_fdma_activate_chan(struct ocelot *ocelot, dma_addr_t dma,
89 ocelot_fdma_writel(ocelot, MSCC_FDMA_DCB_LLP(chan), dma);
90 /* Barrier to force memory writes to DCB to be completed before starting
94 ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_ACTIVATE, BIT(chan));
97 static u32 ocelot_fdma_read_ch_safe(struct ocelot *ocelot)
99 return ocelot_fdma_readl(ocelot, MSCC_FDMA_CH_SAFE);
102 static int ocelot_fdma_wait_chan_safe(struct ocelot *ocelot, int chan)
106 return readx_poll_timeout_atomic(ocelot_fdma_read_ch_safe, ocelot, safe,
108 OCELOT_FDMA_CH_SAFE_TIMEOUT_US);
111 static void ocelot_fdma_dcb_set_data(struct ocelot_fdma_dcb *dcb,
115 u32 offset = dma_addr & 0x3;
118 dcb->datap = ALIGN_DOWN(dma_addr, 4);
119 dcb->datal = ALIGN_DOWN(size, 4);
120 dcb->stat = MSCC_FDMA_DCB_STAT_BLOCKO(offset);
123 static bool ocelot_fdma_rx_alloc_page(struct ocelot *ocelot,
124 struct ocelot_fdma_rx_buf *rxb)
129 page = dev_alloc_page();
133 mapping = dma_map_page(ocelot->dev, page, 0, PAGE_SIZE,
135 if (unlikely(dma_mapping_error(ocelot->dev, mapping))) {
141 rxb->page_offset = 0;
142 rxb->dma_addr = mapping;
147 static int ocelot_fdma_alloc_rx_buffs(struct ocelot *ocelot, u16 alloc_cnt)
149 struct ocelot_fdma *fdma = ocelot->fdma;
150 struct ocelot_fdma_rx_ring *rx_ring;
151 struct ocelot_fdma_rx_buf *rxb;
152 struct ocelot_fdma_dcb *dcb;
157 rx_ring = &fdma->rx_ring;
158 idx = rx_ring->next_to_use;
160 while (alloc_cnt--) {
161 rxb = &rx_ring->bufs[idx];
163 if (unlikely(!rxb->page)) {
164 if (unlikely(!ocelot_fdma_rx_alloc_page(ocelot, rxb))) {
165 dev_err_ratelimited(ocelot->dev,
166 "Failed to allocate rx\n");
172 dcb = &rx_ring->dcbs[idx];
173 dma_addr = rxb->dma_addr + rxb->page_offset;
174 ocelot_fdma_dcb_set_data(dcb, dma_addr, OCELOT_FDMA_RXB_SIZE);
176 idx = ocelot_fdma_idx_next(idx, OCELOT_FDMA_RX_RING_SIZE);
177 /* Chain the DCB to the next one */
178 dcb->llp = ocelot_fdma_idx_dma(rx_ring->dcbs_dma, idx);
181 rx_ring->next_to_use = idx;
182 rx_ring->next_to_alloc = idx;
187 static bool ocelot_fdma_tx_dcb_set_skb(struct ocelot *ocelot,
188 struct ocelot_fdma_tx_buf *tx_buf,
189 struct ocelot_fdma_dcb *dcb,
194 mapping = dma_map_single(ocelot->dev, skb->data, skb->len,
196 if (unlikely(dma_mapping_error(ocelot->dev, mapping)))
199 dma_unmap_addr_set(tx_buf, dma_addr, mapping);
201 ocelot_fdma_dcb_set_data(dcb, mapping, OCELOT_FDMA_RX_SIZE);
203 dcb->stat |= MSCC_FDMA_DCB_STAT_BLOCKL(skb->len);
204 dcb->stat |= MSCC_FDMA_DCB_STAT_SOF | MSCC_FDMA_DCB_STAT_EOF;
209 static bool ocelot_fdma_check_stop_rx(struct ocelot *ocelot)
213 /* Check if the FDMA hits the DCB with LLP == NULL */
214 llp = ocelot_fdma_readl(ocelot, MSCC_FDMA_DCB_LLP(MSCC_FDMA_XTR_CHAN));
218 ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_DISABLE,
219 BIT(MSCC_FDMA_XTR_CHAN));
224 static void ocelot_fdma_rx_set_llp(struct ocelot_fdma_rx_ring *rx_ring)
226 struct ocelot_fdma_dcb *dcb;
229 idx = ocelot_fdma_idx_prev(rx_ring->next_to_use,
230 OCELOT_FDMA_RX_RING_SIZE);
231 dcb = &rx_ring->dcbs[idx];
235 static void ocelot_fdma_rx_restart(struct ocelot *ocelot)
237 struct ocelot_fdma *fdma = ocelot->fdma;
238 struct ocelot_fdma_rx_ring *rx_ring;
239 const u8 chan = MSCC_FDMA_XTR_CHAN;
240 dma_addr_t new_llp, dma_base;
245 rx_ring = &fdma->rx_ring;
246 ret = ocelot_fdma_wait_chan_safe(ocelot, chan);
248 dev_err_ratelimited(ocelot->dev,
249 "Unable to stop RX channel\n");
253 ocelot_fdma_rx_set_llp(rx_ring);
255 /* FDMA stopped on the last DCB that contained a NULL LLP, since
256 * we processed some DCBs in RX, there is free space, and we must set
257 * DCB_LLP to point to the next DCB
259 llp_prev = ocelot_fdma_readl(ocelot, MSCC_FDMA_DCB_LLP_PREV(chan));
260 dma_base = rx_ring->dcbs_dma;
262 /* Get the next DMA addr located after LLP == NULL DCB */
263 idx = ocelot_fdma_dma_idx(dma_base, llp_prev);
264 idx = ocelot_fdma_idx_next(idx, OCELOT_FDMA_RX_RING_SIZE);
265 new_llp = ocelot_fdma_idx_dma(dma_base, idx);
267 /* Finally reactivate the channel */
268 ocelot_fdma_activate_chan(ocelot, new_llp, chan);
271 static bool ocelot_fdma_add_rx_frag(struct ocelot_fdma_rx_buf *rxb, u32 stat,
272 struct sk_buff *skb, bool first)
274 int size = MSCC_FDMA_DCB_STAT_BLOCKL(stat);
275 struct page *page = rxb->page;
280 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
281 rxb->page_offset, size, OCELOT_FDMA_RX_SIZE);
284 /* Try to reuse page */
285 if (unlikely(page_ref_count(page) != 1 || page_is_pfmemalloc(page)))
288 /* Change offset to the other half */
289 rxb->page_offset ^= OCELOT_FDMA_RX_SIZE;
296 static void ocelot_fdma_reuse_rx_page(struct ocelot *ocelot,
297 struct ocelot_fdma_rx_buf *old_rxb)
299 struct ocelot_fdma_rx_ring *rx_ring = &ocelot->fdma->rx_ring;
300 struct ocelot_fdma_rx_buf *new_rxb;
302 new_rxb = &rx_ring->bufs[rx_ring->next_to_alloc];
303 rx_ring->next_to_alloc = ocelot_fdma_idx_next(rx_ring->next_to_alloc,
304 OCELOT_FDMA_RX_RING_SIZE);
306 /* Copy page reference */
309 /* Sync for use by the device */
310 dma_sync_single_range_for_device(ocelot->dev, old_rxb->dma_addr,
311 old_rxb->page_offset,
312 OCELOT_FDMA_RX_SIZE, DMA_FROM_DEVICE);
315 static struct sk_buff *ocelot_fdma_get_skb(struct ocelot *ocelot, u32 stat,
316 struct ocelot_fdma_rx_buf *rxb,
321 /* Allocate skb head and data */
323 void *buff_addr = page_address(rxb->page) +
326 skb = build_skb(buff_addr, OCELOT_FDMA_SKBFRAG_SIZE);
327 if (unlikely(!skb)) {
328 dev_err_ratelimited(ocelot->dev,
329 "build_skb failed !\n");
335 dma_sync_single_range_for_cpu(ocelot->dev, rxb->dma_addr,
336 rxb->page_offset, OCELOT_FDMA_RX_SIZE,
339 if (ocelot_fdma_add_rx_frag(rxb, stat, skb, first)) {
340 /* Reuse the free half of the page for the next_to_alloc DCB*/
341 ocelot_fdma_reuse_rx_page(ocelot, rxb);
343 /* page cannot be reused, unmap it */
344 dma_unmap_page(ocelot->dev, rxb->dma_addr, PAGE_SIZE,
348 /* clear rx buff content */
354 static bool ocelot_fdma_receive_skb(struct ocelot *ocelot, struct sk_buff *skb)
356 struct net_device *ndev;
357 void *xfh = skb->data;
361 skb_pull(skb, OCELOT_TAG_LEN);
363 ocelot_xfh_get_src_port(xfh, &src_port);
364 if (unlikely(src_port >= ocelot->num_phys_ports))
367 ndev = ocelot_port_to_netdev(ocelot, src_port);
371 if (pskb_trim(skb, skb->len - ETH_FCS_LEN))
375 skb->protocol = eth_type_trans(skb, skb->dev);
376 skb->dev->stats.rx_bytes += skb->len;
377 skb->dev->stats.rx_packets++;
380 ocelot_xfh_get_rew_val(xfh, ×tamp);
381 ocelot_ptp_rx_timestamp(ocelot, skb, timestamp);
384 if (likely(!skb_defer_rx_timestamp(skb)))
385 netif_receive_skb(skb);
390 static int ocelot_fdma_rx_get(struct ocelot *ocelot, int budget)
392 struct ocelot_fdma *fdma = ocelot->fdma;
393 struct ocelot_fdma_rx_ring *rx_ring;
394 struct ocelot_fdma_rx_buf *rxb;
395 struct ocelot_fdma_dcb *dcb;
402 cleaned_cnt = ocelot_fdma_rx_ring_free(fdma);
403 rx_ring = &fdma->rx_ring;
407 idx = rx_ring->next_to_clean;
408 dcb = &rx_ring->dcbs[idx];
410 if (MSCC_FDMA_DCB_STAT_BLOCKL(stat) == 0)
413 /* New packet is a start of frame but we already got a skb set,
414 * we probably lost an EOF packet, free skb
416 if (unlikely(skb && (stat & MSCC_FDMA_DCB_STAT_SOF))) {
421 rxb = &rx_ring->bufs[idx];
422 /* Fetch next to clean buffer from the rx_ring */
423 skb = ocelot_fdma_get_skb(ocelot, stat, rxb, skb);
430 idx = ocelot_fdma_idx_next(idx, OCELOT_FDMA_RX_RING_SIZE);
431 rx_ring->next_to_clean = idx;
433 if (unlikely(stat & MSCC_FDMA_DCB_STAT_ABORT ||
434 stat & MSCC_FDMA_DCB_STAT_PD)) {
435 dev_err_ratelimited(ocelot->dev,
436 "DCB aborted or pruned\n");
442 /* We still need to process the other fragment of the packet
443 * before delivering it to the network stack
445 if (!(stat & MSCC_FDMA_DCB_STAT_EOF))
448 if (unlikely(!ocelot_fdma_receive_skb(ocelot, skb)))
457 ocelot_fdma_alloc_rx_buffs(ocelot, cleaned_cnt);
462 static void ocelot_fdma_wakeup_netdev(struct ocelot *ocelot)
464 struct ocelot_port_private *priv;
465 struct ocelot_port *ocelot_port;
466 struct net_device *dev;
469 for (port = 0; port < ocelot->num_phys_ports; port++) {
470 ocelot_port = ocelot->ports[port];
473 priv = container_of(ocelot_port, struct ocelot_port_private,
477 if (unlikely(netif_queue_stopped(dev)))
478 netif_wake_queue(dev);
482 static void ocelot_fdma_tx_cleanup(struct ocelot *ocelot, int budget)
484 struct ocelot_fdma *fdma = ocelot->fdma;
485 struct ocelot_fdma_tx_ring *tx_ring;
486 struct ocelot_fdma_tx_buf *buf;
487 unsigned int new_null_llp_idx;
488 struct ocelot_fdma_dcb *dcb;
489 bool end_of_list = false;
496 tx_ring = &fdma->tx_ring;
498 /* Purge the TX packets that have been sent up to the NULL llp or the
501 while (!ocelot_fdma_tx_ring_empty(fdma)) {
502 ntc = tx_ring->next_to_clean;
503 dcb = &tx_ring->dcbs[ntc];
504 if (!(dcb->stat & MSCC_FDMA_DCB_STAT_PD))
507 buf = &tx_ring->bufs[ntc];
509 dma_unmap_single(ocelot->dev, dma_unmap_addr(buf, dma_addr),
510 skb->len, DMA_TO_DEVICE);
511 napi_consume_skb(skb, budget);
514 /* Only update after accessing all dcb fields */
515 tx_ring->next_to_clean = ocelot_fdma_idx_next(ntc,
516 OCELOT_FDMA_TX_RING_SIZE);
518 /* If we hit the NULL LLP, stop, we might need to reload FDMA */
525 /* No need to try to wake if there were no TX cleaned_cnt up. */
526 if (ocelot_fdma_tx_ring_free(fdma))
527 ocelot_fdma_wakeup_netdev(ocelot);
529 /* If there is still some DCBs to be processed by the FDMA or if the
530 * pending list is empty, there is no need to restart the FDMA.
532 if (!end_of_list || ocelot_fdma_tx_ring_empty(fdma))
535 ret = ocelot_fdma_wait_chan_safe(ocelot, MSCC_FDMA_INJ_CHAN);
537 dev_warn(ocelot->dev,
538 "Failed to wait for TX channel to stop\n");
542 /* Set NULL LLP to be the last DCB used */
543 new_null_llp_idx = ocelot_fdma_idx_prev(tx_ring->next_to_use,
544 OCELOT_FDMA_TX_RING_SIZE);
545 dcb = &tx_ring->dcbs[new_null_llp_idx];
548 dma = ocelot_fdma_idx_dma(tx_ring->dcbs_dma, tx_ring->next_to_clean);
549 ocelot_fdma_activate_chan(ocelot, dma, MSCC_FDMA_INJ_CHAN);
552 static int ocelot_fdma_napi_poll(struct napi_struct *napi, int budget)
554 struct ocelot_fdma *fdma = container_of(napi, struct ocelot_fdma, napi);
555 struct ocelot *ocelot = fdma->ocelot;
559 ocelot_fdma_tx_cleanup(ocelot, budget);
561 rx_stopped = ocelot_fdma_check_stop_rx(ocelot);
563 work_done = ocelot_fdma_rx_get(ocelot, budget);
566 ocelot_fdma_rx_restart(ocelot);
568 if (work_done < budget) {
569 napi_complete_done(&fdma->napi, work_done);
570 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA,
571 BIT(MSCC_FDMA_INJ_CHAN) |
572 BIT(MSCC_FDMA_XTR_CHAN));
578 static irqreturn_t ocelot_fdma_interrupt(int irq, void *dev_id)
580 u32 ident, llp, frm, err, err_code;
581 struct ocelot *ocelot = dev_id;
583 ident = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_IDENT);
584 frm = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_FRM);
585 llp = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_LLP);
587 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP, llp & ident);
588 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM, frm & ident);
590 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);
591 napi_schedule(&ocelot->fdma->napi);
594 err = ocelot_fdma_readl(ocelot, MSCC_FDMA_EVT_ERR);
596 err_code = ocelot_fdma_readl(ocelot, MSCC_FDMA_EVT_ERR_CODE);
597 dev_err_ratelimited(ocelot->dev,
598 "Error ! chans mask: %#x, code: %#x\n",
601 ocelot_fdma_writel(ocelot, MSCC_FDMA_EVT_ERR, err);
602 ocelot_fdma_writel(ocelot, MSCC_FDMA_EVT_ERR_CODE, err_code);
608 static void ocelot_fdma_send_skb(struct ocelot *ocelot,
609 struct ocelot_fdma *fdma, struct sk_buff *skb)
611 struct ocelot_fdma_tx_ring *tx_ring = &fdma->tx_ring;
612 struct ocelot_fdma_tx_buf *tx_buf;
613 struct ocelot_fdma_dcb *dcb;
617 dcb = &tx_ring->dcbs[tx_ring->next_to_use];
618 tx_buf = &tx_ring->bufs[tx_ring->next_to_use];
619 if (!ocelot_fdma_tx_dcb_set_skb(ocelot, tx_buf, dcb, skb)) {
620 dev_kfree_skb_any(skb);
624 next_idx = ocelot_fdma_idx_next(tx_ring->next_to_use,
625 OCELOT_FDMA_TX_RING_SIZE);
626 skb_tx_timestamp(skb);
628 /* If the FDMA TX chan is empty, then enqueue the DCB directly */
629 if (ocelot_fdma_tx_ring_empty(fdma)) {
630 dma = ocelot_fdma_idx_dma(tx_ring->dcbs_dma,
631 tx_ring->next_to_use);
632 ocelot_fdma_activate_chan(ocelot, dma, MSCC_FDMA_INJ_CHAN);
635 dcb->llp = ocelot_fdma_idx_dma(tx_ring->dcbs_dma, next_idx);
638 tx_ring->next_to_use = next_idx;
641 static int ocelot_fdma_prepare_skb(struct ocelot *ocelot, int port, u32 rew_op,
642 struct sk_buff *skb, struct net_device *dev)
644 int needed_headroom = max_t(int, OCELOT_TAG_LEN - skb_headroom(skb), 0);
645 int needed_tailroom = max_t(int, ETH_FCS_LEN - skb_tailroom(skb), 0);
649 if (unlikely(needed_headroom || needed_tailroom ||
650 skb_header_cloned(skb))) {
651 err = pskb_expand_head(skb, needed_headroom, needed_tailroom,
654 dev_kfree_skb_any(skb);
659 err = skb_linearize(skb);
661 net_err_ratelimited("%s: skb_linearize error (%d)!\n",
663 dev_kfree_skb_any(skb);
667 ifh = skb_push(skb, OCELOT_TAG_LEN);
668 skb_put(skb, ETH_FCS_LEN);
669 memset(ifh, 0, OCELOT_TAG_LEN);
670 ocelot_ifh_port_set(ifh, port, rew_op, skb_vlan_tag_get(skb));
675 int ocelot_fdma_inject_frame(struct ocelot *ocelot, int port, u32 rew_op,
676 struct sk_buff *skb, struct net_device *dev)
678 struct ocelot_fdma *fdma = ocelot->fdma;
679 int ret = NETDEV_TX_OK;
681 spin_lock(&fdma->tx_ring.xmit_lock);
683 if (ocelot_fdma_tx_ring_free(fdma) == 0) {
684 netif_stop_queue(dev);
685 ret = NETDEV_TX_BUSY;
689 if (ocelot_fdma_prepare_skb(ocelot, port, rew_op, skb, dev))
692 ocelot_fdma_send_skb(ocelot, fdma, skb);
695 spin_unlock(&fdma->tx_ring.xmit_lock);
700 static void ocelot_fdma_free_rx_ring(struct ocelot *ocelot)
702 struct ocelot_fdma *fdma = ocelot->fdma;
703 struct ocelot_fdma_rx_ring *rx_ring;
704 struct ocelot_fdma_rx_buf *rxb;
707 rx_ring = &fdma->rx_ring;
708 idx = rx_ring->next_to_clean;
710 /* Free the pages held in the RX ring */
711 while (idx != rx_ring->next_to_use) {
712 rxb = &rx_ring->bufs[idx];
713 dma_unmap_page(ocelot->dev, rxb->dma_addr, PAGE_SIZE,
715 __free_page(rxb->page);
716 idx = ocelot_fdma_idx_next(idx, OCELOT_FDMA_RX_RING_SIZE);
719 if (fdma->rx_ring.skb)
720 dev_kfree_skb_any(fdma->rx_ring.skb);
723 static void ocelot_fdma_free_tx_ring(struct ocelot *ocelot)
725 struct ocelot_fdma *fdma = ocelot->fdma;
726 struct ocelot_fdma_tx_ring *tx_ring;
727 struct ocelot_fdma_tx_buf *txb;
731 tx_ring = &fdma->tx_ring;
732 idx = tx_ring->next_to_clean;
734 while (idx != tx_ring->next_to_use) {
735 txb = &tx_ring->bufs[idx];
737 dma_unmap_single(ocelot->dev, dma_unmap_addr(txb, dma_addr),
738 skb->len, DMA_TO_DEVICE);
739 dev_kfree_skb_any(skb);
740 idx = ocelot_fdma_idx_next(idx, OCELOT_FDMA_TX_RING_SIZE);
744 static int ocelot_fdma_rings_alloc(struct ocelot *ocelot)
746 struct ocelot_fdma *fdma = ocelot->fdma;
747 struct ocelot_fdma_dcb *dcbs;
752 /* Create a pool of consistent memory blocks for hardware descriptors */
753 fdma->dcbs_base = dmam_alloc_coherent(ocelot->dev,
754 OCELOT_DCBS_HW_ALLOC_SIZE,
755 &fdma->dcbs_dma_base, GFP_KERNEL);
756 if (!fdma->dcbs_base)
759 /* DCBs must be aligned on a 32bit boundary */
760 dcbs = fdma->dcbs_base;
761 dcbs_dma = fdma->dcbs_dma_base;
762 if (!IS_ALIGNED(dcbs_dma, 4)) {
763 adjust = dcbs_dma & 0x3;
764 dcbs_dma = ALIGN(dcbs_dma, 4);
765 dcbs = (void *)dcbs + adjust;
769 fdma->tx_ring.dcbs = dcbs;
770 fdma->tx_ring.dcbs_dma = dcbs_dma;
771 spin_lock_init(&fdma->tx_ring.xmit_lock);
774 fdma->rx_ring.dcbs = dcbs + OCELOT_FDMA_TX_RING_SIZE;
775 fdma->rx_ring.dcbs_dma = dcbs_dma + OCELOT_FDMA_TX_DCB_SIZE;
776 ret = ocelot_fdma_alloc_rx_buffs(ocelot,
777 ocelot_fdma_tx_ring_free(fdma));
779 ocelot_fdma_free_rx_ring(ocelot);
783 /* Set the last DCB LLP as NULL, this is normally done when restarting
784 * the RX chan, but this is for the first run
786 ocelot_fdma_rx_set_llp(&fdma->rx_ring);
791 void ocelot_fdma_netdev_init(struct ocelot *ocelot, struct net_device *dev)
793 struct ocelot_fdma *fdma = ocelot->fdma;
795 dev->needed_headroom = OCELOT_TAG_LEN;
796 dev->needed_tailroom = ETH_FCS_LEN;
802 netif_napi_add_weight(dev, &fdma->napi, ocelot_fdma_napi_poll,
806 void ocelot_fdma_netdev_deinit(struct ocelot *ocelot, struct net_device *dev)
808 struct ocelot_fdma *fdma = ocelot->fdma;
810 if (fdma->ndev == dev) {
811 netif_napi_del(&fdma->napi);
816 void ocelot_fdma_init(struct platform_device *pdev, struct ocelot *ocelot)
818 struct device *dev = ocelot->dev;
819 struct ocelot_fdma *fdma;
822 fdma = devm_kzalloc(dev, sizeof(*fdma), GFP_KERNEL);
827 ocelot->dev->coherent_dma_mask = DMA_BIT_MASK(32);
829 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);
831 fdma->ocelot = ocelot;
832 fdma->irq = platform_get_irq_byname(pdev, "fdma");
833 ret = devm_request_irq(dev, fdma->irq, ocelot_fdma_interrupt, 0,
834 dev_name(dev), ocelot);
838 ret = ocelot_fdma_rings_alloc(ocelot);
842 static_branch_enable(&ocelot_fdma_enabled);
847 devm_free_irq(dev, fdma->irq, fdma);
849 devm_kfree(dev, fdma);
854 void ocelot_fdma_start(struct ocelot *ocelot)
856 struct ocelot_fdma *fdma = ocelot->fdma;
858 /* Reconfigure for extraction and injection using DMA */
859 ocelot_write_rix(ocelot, QS_INJ_GRP_CFG_MODE(2), QS_INJ_GRP_CFG, 0);
860 ocelot_write_rix(ocelot, QS_INJ_CTRL_GAP_SIZE(0), QS_INJ_CTRL, 0);
862 ocelot_write_rix(ocelot, QS_XTR_GRP_CFG_MODE(2), QS_XTR_GRP_CFG, 0);
864 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP, 0xffffffff);
865 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM, 0xffffffff);
867 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP_ENA,
868 BIT(MSCC_FDMA_INJ_CHAN) | BIT(MSCC_FDMA_XTR_CHAN));
869 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM_ENA,
870 BIT(MSCC_FDMA_XTR_CHAN));
871 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA,
872 BIT(MSCC_FDMA_INJ_CHAN) | BIT(MSCC_FDMA_XTR_CHAN));
874 napi_enable(&fdma->napi);
876 ocelot_fdma_activate_chan(ocelot, ocelot->fdma->rx_ring.dcbs_dma,
880 void ocelot_fdma_deinit(struct ocelot *ocelot)
882 struct ocelot_fdma *fdma = ocelot->fdma;
884 ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);
885 ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_FORCEDIS,
886 BIT(MSCC_FDMA_XTR_CHAN));
887 ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_FORCEDIS,
888 BIT(MSCC_FDMA_INJ_CHAN));
889 napi_synchronize(&fdma->napi);
890 napi_disable(&fdma->napi);
892 ocelot_fdma_free_rx_ring(ocelot);
893 ocelot_fdma_free_tx_ring(ocelot);