1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (c) 2006 ARM Ltd.
4 * Copyright (c) 2010 ST-Ericsson SA
5 * Copyirght (c) 2017 Linaro Ltd.
7 * Author: Peter Pearse <peter.pearse@arm.com>
8 * Author: Linus Walleij <linus.walleij@linaro.org>
10 * Documentation: ARM DDI 0196G == PL080
11 * Documentation: ARM DDI 0218E == PL081
12 * Documentation: S3C6410 User's Manual == PL080S
14 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
17 * The PL080 has 8 channels available for simultaneous use, and the PL081
18 * has only two channels. So on these DMA controllers the number of channels
19 * and the number of incoming DMA signals are two totally different things.
20 * It is usually not possible to theoretically handle all physical signals,
21 * so a multiplexing scheme with possible denial of use is necessary.
23 * The PL080 has a dual bus master, PL081 has a single master.
25 * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
26 * It differs in following aspects:
27 * - CH_CONFIG register at different offset,
28 * - separate CH_CONTROL2 register for transfer size,
29 * - bigger maximum transfer size,
30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
31 * - no support for peripheral flow control.
33 * Memory to peripheral transfer may be visualized as
34 * Get data from memory to DMAC
36 * On burst request from peripheral
37 * Destination burst from DMAC to peripheral
39 * Raise terminal count interrupt
41 * For peripherals with a FIFO:
42 * Source burst size == half the depth of the peripheral FIFO
43 * Destination burst size == the depth of the peripheral FIFO
45 * (Bursts are irrelevant for mem to mem transfers - there are no burst
46 * signals, the DMA controller will simply facilitate its AHB master.)
48 * ASSUMES default (little) endianness for DMA transfers
50 * The PL08x has two flow control settings:
51 * - DMAC flow control: the transfer size defines the number of transfers
52 * which occur for the current LLI entry, and the DMAC raises TC at the
53 * end of every LLI entry. Observed behaviour shows the DMAC listening
54 * to both the BREQ and SREQ signals (contrary to documented),
55 * transferring data if either is active. The LBREQ and LSREQ signals
58 * - Peripheral flow control: the transfer size is ignored (and should be
59 * zero). The data is transferred from the current LLI entry, until
60 * after the final transfer signalled by LBREQ or LSREQ. The DMAC
61 * will then move to the next LLI entry. Unsupported by PL080S.
63 //#include <linux/amba/bus.h>
64 #include <linux/amba/pl08x.h>
65 #include <linux/debugfs.h>
66 #include <linux/delay.h>
67 #include <linux/device.h>
68 #include <linux/dmaengine.h>
69 #include <linux/dmapool.h>
70 #include <linux/dma-mapping.h>
71 #include <linux/export.h>
72 #include <linux/init.h>
73 #include <linux/interrupt.h>
74 #include <linux/module.h>
76 #include <linux/of_dma.h>
77 #include <linux/of_device.h>
78 #include <linux/platform_device.h>
79 #include <linux/pm_runtime.h>
80 #include <linux/seq_file.h>
81 #include <linux/slab.h>
82 #include <linux/amba/pl080.h>
84 #include "dmaengine.h"
87 #define DRIVER_NAME "pl08xdmac"
89 #define PL80X_DMA_BUSWIDTHS \
90 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
91 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
92 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
93 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
95 //static struct amba_driver pl08x_amba_driver;
96 struct pl08x_driver_data;
99 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
100 * @config_offset: offset to the configuration register
101 * @channels: the number of channels available in this variant
102 * @signals: the number of request signals available from the hardware
103 * @dualmaster: whether this version supports dual AHB masters or not.
104 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the
105 * channels have Nomadik security extension bits that need to be checked
106 * for permission before use and some registers are missing
107 * @pl080s: whether this variant is a Samsung PL080S, which has separate
108 * register and LLI word for transfer size.
109 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020
110 * @max_transfer_size: the maximum single element transfer size for this
121 u32 max_transfer_size;
125 * struct pl08x_bus_data - information of source or destination
126 * busses for a transfer
127 * @addr: current address
128 * @maxwidth: the maximum width of a transfer on this bus
129 * @buswidth: the width of this bus in bytes: 1, 2 or 4
131 struct pl08x_bus_data {
137 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
140 * struct pl08x_phy_chan - holder for the physical channels
141 * @id: physical index to this channel
142 * @base: memory base address for this physical channel
143 * @reg_config: configuration address for this physical channel
144 * @reg_control: control address for this physical channel
145 * @reg_src: transfer source address register
146 * @reg_dst: transfer destination address register
147 * @reg_lli: transfer LLI address register
148 * @reg_busy: if the variant has a special per-channel busy register,
149 * this contains a pointer to it
150 * @lock: a lock to use when altering an instance of this struct
151 * @serving: the virtual channel currently being served by this physical
153 * @locked: channel unavailable for the system, e.g. dedicated to secure
155 * @ftdmac020: channel is on a FTDMAC020
156 * @pl080s: channel is on a PL08s
158 struct pl08x_phy_chan {
161 void __iomem *reg_config;
162 void __iomem *reg_control;
163 void __iomem *reg_src;
164 void __iomem *reg_dst;
165 void __iomem *reg_lli;
166 void __iomem *reg_busy;
168 struct pl08x_dma_chan *serving;
175 * struct pl08x_sg - structure containing data per sg
176 * @src_addr: src address of sg
177 * @dst_addr: dst address of sg
178 * @len: transfer len in bytes
179 * @node: node for txd's dsg_list
185 struct list_head node;
189 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
190 * @vd: virtual DMA descriptor
191 * @dsg_list: list of children sg's
192 * @llis_bus: DMA memory address (physical) start for the LLIs
193 * @llis_va: virtual memory address start for the LLIs
194 * @cctl: control reg values for current txd
195 * @ccfg: config reg values for current txd
196 * @done: this marks completed descriptors, which should not have their
198 * @cyclic: indicate cyclic transfers
201 struct virt_dma_desc vd;
202 struct list_head dsg_list;
205 /* Default cctl value for LLIs */
208 * Settings to be put into the physical channel when we
209 * trigger this txd. Other registers are in llis_va[0].
217 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
219 * @PL08X_CHAN_IDLE: the channel is idle
220 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
221 * channel and is running a transfer on it
222 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
223 * channel, but the transfer is currently paused
224 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
225 * channel to become available (only pertains to memcpy channels)
227 enum pl08x_dma_chan_state {
235 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
236 * @vc: wrappped virtual channel
237 * @phychan: the physical channel utilized by this channel, if there is one
238 * @name: name of channel
239 * @cd: channel platform data
240 * @cfg: slave configuration
241 * @at: active transaction on this channel
242 * @host: a pointer to the host (internal use)
243 * @state: whether the channel is idle, paused, running etc
244 * @slave: whether this channel is a device (slave) or for memcpy
245 * @signal: the physical DMA request signal which this channel is using
246 * @mux_use: count of descriptors using this DMA request signal setting
247 * @waiting_at: time in jiffies when this channel moved to waiting state
249 struct pl08x_dma_chan {
250 struct virt_dma_chan vc;
251 struct pl08x_phy_chan *phychan;
253 struct pl08x_channel_data *cd;
254 struct dma_slave_config cfg;
255 struct pl08x_txd *at;
256 struct pl08x_driver_data *host;
257 enum pl08x_dma_chan_state state;
262 unsigned long waiting_at;
266 * struct pl08x_driver_data - the local state holder for the PL08x
267 * @slave: optional slave engine for this instance
268 * @memcpy: memcpy engine for this instance
269 * @has_slave: the PL08x has a slave engine (routed signals)
270 * @base: virtual memory base (remapped) for the PL08x
271 * @adev: the corresponding AMBA (PrimeCell) bus entry
272 * @vd: vendor data for this PL08x variant
273 * @pd: platform data passed in from the platform/machine
274 * @phy_chans: array of data for the physical channels
275 * @pool: a pool for the LLI descriptors
276 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
278 * @mem_buses: set to indicate memory transfers on AHB2.
279 * @lli_words: how many words are used in each LLI item for this variant
281 struct pl08x_driver_data {
282 struct dma_device slave;
283 struct dma_device memcpy;
286 struct platform_device *adev;
287 const struct vendor_data *vd;
288 struct pl08x_platform_data *pd;
289 struct pl08x_phy_chan *phy_chans;
290 struct dma_pool *pool;
297 * PL08X specific defines
300 /* The order of words in an LLI. */
301 #define PL080_LLI_SRC 0
302 #define PL080_LLI_DST 1
303 #define PL080_LLI_LLI 2
304 #define PL080_LLI_CCTL 3
305 #define PL080S_LLI_CCTL2 4
307 /* Total words in an LLI. */
308 #define PL080_LLI_WORDS 4
309 #define PL080S_LLI_WORDS 8
312 * Number of LLIs in each LLI buffer allocated for one transfer
313 * (maximum times we call dma_pool_alloc on this pool without freeing)
315 #define MAX_NUM_TSFR_LLIS 512
316 #define PL08X_ALIGN 8
318 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
320 return container_of(chan, struct pl08x_dma_chan, vc.chan);
323 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
325 return container_of(tx, struct pl08x_txd, vd.tx);
331 * This gives us the DMA request input to the PL08x primecell which the
332 * peripheral described by the channel data will be routed to, possibly
333 * via a board/SoC specific external MUX. One important point to note
334 * here is that this does not depend on the physical channel.
336 static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
338 const struct pl08x_platform_data *pd = plchan->host->pd;
341 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
342 ret = pd->get_xfer_signal(plchan->cd);
348 plchan->signal = ret;
353 static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
355 const struct pl08x_platform_data *pd = plchan->host->pd;
357 if (plchan->signal >= 0) {
358 WARN_ON(plchan->mux_use == 0);
360 if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
361 pd->put_xfer_signal(plchan->cd, plchan->signal);
368 * Physical channel handling
371 /* Whether a certain channel is busy or not */
372 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
376 /* If we have a special busy register, take a shortcut */
378 val = readl(ch->reg_busy);
379 return !!(val & BIT(ch->id));
381 val = readl(ch->reg_config);
382 return val & PL080_CONFIG_ACTIVE;
386 * pl08x_write_lli() - Write an LLI into the DMA controller.
388 * The PL08x derivatives support linked lists, but the first item of the
389 * list containing the source, destination, control word and next LLI is
390 * ignored. Instead the driver has to write those values directly into the
391 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
392 * register need to be set up for the first transfer.
394 static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
395 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
397 if (pl08x->vd->pl080s)
398 dev_vdbg(&pl08x->adev->dev,
399 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
400 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
401 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
402 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
403 lli[PL080S_LLI_CCTL2], ccfg);
405 //dev_vdbg(&pl08x->adev->dev,
406 dev_info(&pl08x->adev->dev,
407 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
408 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
409 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
410 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
412 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);
413 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);
414 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);
417 * The FTMAC020 has a different layout in the CCTL word of the LLI
418 * and the CCTL register which is split in CSR and SIZE registers.
419 * Convert the LLI item CCTL into the proper values to write into
420 * the CSR and SIZE registers.
422 if (phychan->ftdmac020) {
423 u32 llictl = lli[PL080_LLI_CCTL];
426 /* Write the transfer size (12 bits) to the size register */
427 writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,
428 phychan->base + FTDMAC020_CH_SIZE);
430 * Then write the control bits 28..16 to the control register
431 * by shuffleing the bits around to where they are in the
432 * main register. The mapping is as follows:
433 * Bit 28: TC_MSK - mask on all except last LLI
434 * Bit 27..25: SRC_WIDTH
435 * Bit 24..22: DST_WIDTH
436 * Bit 21..20: SRCAD_CTRL
437 * Bit 19..17: DSTAD_CTRL
441 if (llictl & FTDMAC020_LLI_TC_MSK)
442 val |= FTDMAC020_CH_CSR_TC_MSK;
443 val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
444 (FTDMAC020_LLI_SRC_WIDTH_SHIFT -
445 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));
446 val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
447 (FTDMAC020_LLI_DST_WIDTH_SHIFT -
448 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));
449 val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >>
450 (FTDMAC020_LLI_SRCAD_CTL_SHIFT -
451 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));
452 val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >>
453 (FTDMAC020_LLI_DSTAD_CTL_SHIFT -
454 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));
455 if (llictl & FTDMAC020_LLI_SRC_SEL)
456 val |= FTDMAC020_CH_CSR_SRC_SEL;
457 if (llictl & FTDMAC020_LLI_DST_SEL)
458 val |= FTDMAC020_CH_CSR_DST_SEL;
461 * Set up the bits that exist in the CSR but are not
462 * part the LLI, i.e. only gets written to the control
463 * register right here.
465 * FIXME: do not just handle memcpy, also handle slave DMA.
467 switch (pl08x->pd->memcpy_burst_size) {
469 case PL08X_BURST_SZ_1:
470 val |= PL080_BSIZE_1 <<
471 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
473 case PL08X_BURST_SZ_4:
474 val |= PL080_BSIZE_4 <<
475 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
477 case PL08X_BURST_SZ_8:
478 val |= PL080_BSIZE_8 <<
479 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
481 case PL08X_BURST_SZ_16:
482 val |= PL080_BSIZE_16 <<
483 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
485 case PL08X_BURST_SZ_32:
486 val |= PL080_BSIZE_32 <<
487 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
489 case PL08X_BURST_SZ_64:
490 val |= PL080_BSIZE_64 <<
491 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
493 case PL08X_BURST_SZ_128:
494 val |= PL080_BSIZE_128 <<
495 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
497 case PL08X_BURST_SZ_256:
498 val |= PL080_BSIZE_256 <<
499 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
503 /* Protection flags */
504 if (pl08x->pd->memcpy_prot_buff)
505 val |= FTDMAC020_CH_CSR_PROT2;
506 if (pl08x->pd->memcpy_prot_cache)
507 val |= FTDMAC020_CH_CSR_PROT3;
508 /* We are the kernel, so we are in privileged mode */
509 val |= FTDMAC020_CH_CSR_PROT1;
511 writel_relaxed(val, phychan->reg_control);
513 /* printk("this is debug lli[PL080_LLI_CCTL] = %x reg_control = %x %s %s %d\n",
514 lli[PL080_LLI_CCTL],phychan->reg_control,__FILE__,__func__,__LINE__);
516 /* Bits are just identical */
517 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);
520 /* Second control word on the PL080s */
521 if (pl08x->vd->pl080s)
522 writel_relaxed(lli[PL080S_LLI_CCTL2],
523 phychan->base + PL080S_CH_CONTROL2);
526 printk("this is debug ccfg = %x reg_config = %x %s %s %d\n",
527 ccfg,phychan->reg_config,__FILE__,__func__,__LINE__);
529 writel(ccfg, phychan->reg_config);
533 * Set the initial DMA register values i.e. those for the first LLI
534 * The next LLI pointer and the configuration interrupt bit have
535 * been set when the LLIs were constructed. Poke them into the hardware
536 * and start the transfer.
538 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
540 struct pl08x_driver_data *pl08x = plchan->host;
541 struct pl08x_phy_chan *phychan = plchan->phychan;
542 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
543 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
546 list_del(&txd->vd.node);
550 /* Wait for channel inactive */
551 while (pl08x_phy_channel_busy(phychan))
553 //printk("this is debug txd->ccfg = %x %s %s %d\n",txd->ccfg,__FILE__,__func__,__LINE__);
554 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
556 /* Enable the DMA channel */
557 /* Do not access config register until channel shows as disabled */
558 //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
559 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))
562 //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
563 /* Do not access config register until channel shows as inactive */
564 if (phychan->ftdmac020) {
565 val = readl(phychan->reg_config);
566 while (val & FTDMAC020_CH_CFG_BUSY)
567 val = readl(phychan->reg_config);
569 val = readl(phychan->reg_control);
570 while (val & FTDMAC020_CH_CSR_EN)
571 val = readl(phychan->reg_control);
573 writel(val | FTDMAC020_CH_CSR_EN,
574 phychan->reg_control);
576 val = readl(phychan->reg_config);
577 while ((val & PL080_CONFIG_ACTIVE) ||
578 (val & PL080_CONFIG_ENABLE))
579 val = readl(phychan->reg_config);
580 //printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__);
581 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
583 while(!(readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))){
584 printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__);
585 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
589 //printk("this is debug reg_config = %x en_chan = %x id = %d %s %s %d\n",readl(phychan->reg_config),
590 // readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
594 * Pause the channel by setting the HALT bit.
596 * For M->P transfers, pause the DMAC first and then stop the peripheral -
597 * the FIFO can only drain if the peripheral is still requesting data.
598 * (note: this can still timeout if the DMAC FIFO never drains of data.)
600 * For P->M transfers, disable the peripheral first to stop it filling
601 * the DMAC FIFO, and then pause the DMAC.
603 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
609 /* Use the enable bit on the FTDMAC020 */
610 val = readl(ch->reg_control);
611 val &= ~FTDMAC020_CH_CSR_EN;
612 writel(val, ch->reg_control);
616 /* Set the HALT bit and wait for the FIFO to drain */
617 val = readl(ch->reg_config);
618 val |= PL080_CONFIG_HALT;
619 writel(val, ch->reg_config);
621 /* Wait for channel inactive */
622 for (timeout = 1000; timeout; timeout--) {
623 if (!pl08x_phy_channel_busy(ch))
627 if (pl08x_phy_channel_busy(ch))
628 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
631 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
635 /* Use the enable bit on the FTDMAC020 */
637 val = readl(ch->reg_control);
638 val |= FTDMAC020_CH_CSR_EN;
639 writel(val, ch->reg_control);
643 /* Clear the HALT bit */
644 val = readl(ch->reg_config);
645 val &= ~PL080_CONFIG_HALT;
646 writel(val, ch->reg_config);
650 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
651 * clears any pending interrupt status. This should not be used for
652 * an on-going transfer, but as a method of shutting down a channel
653 * (eg, when it's no longer used) or terminating a transfer.
655 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
656 struct pl08x_phy_chan *ch)
660 /* The layout for the FTDMAC020 is different */
662 /* Disable all interrupts */
663 val = readl(ch->reg_config);
664 val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |
665 FTDMAC020_CH_CFG_INT_ERR_MASK |
666 FTDMAC020_CH_CFG_INT_TC_MASK);
667 writel(val, ch->reg_config);
669 /* Abort and disable channel */
670 val = readl(ch->reg_control);
671 val &= ~FTDMAC020_CH_CSR_EN;
672 val |= FTDMAC020_CH_CSR_ABT;
673 writel(val, ch->reg_control);
675 /* Clear ABT and ERR interrupt flags */
676 writel(BIT(ch->id) | BIT(ch->id + 16),
677 pl08x->base + PL080_ERR_CLEAR);
678 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
683 val = readl(ch->reg_config);
684 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
685 PL080_CONFIG_TC_IRQ_MASK);
686 writel(val, ch->reg_config);
688 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR);
689 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
692 static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)
698 bytes = readl(ch->base + FTDMAC020_CH_SIZE);
700 val = readl(ch->reg_control);
701 val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;
702 val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;
703 } else if (ch->pl080s) {
704 val = readl(ch->base + PL080S_CH_CONTROL2);
705 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
707 val = readl(ch->reg_control);
708 val &= PL080_CONTROL_SWIDTH_MASK;
709 val >>= PL080_CONTROL_SWIDTH_SHIFT;
712 val = readl(ch->reg_control);
713 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
715 val &= PL080_CONTROL_SWIDTH_MASK;
716 val >>= PL080_CONTROL_SWIDTH_SHIFT;
720 case PL080_WIDTH_8BIT:
722 case PL080_WIDTH_16BIT:
725 case PL080_WIDTH_32BIT:
732 static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)
738 val = llis_va[PL080_LLI_CCTL];
739 bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;
741 val = llis_va[PL080_LLI_CCTL];
742 val &= FTDMAC020_LLI_SRC_WIDTH_MSK;
743 val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;
744 } else if (ch->pl080s) {
745 val = llis_va[PL080S_LLI_CCTL2];
746 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
748 val = llis_va[PL080_LLI_CCTL];
749 val &= PL080_CONTROL_SWIDTH_MASK;
750 val >>= PL080_CONTROL_SWIDTH_SHIFT;
753 val = llis_va[PL080_LLI_CCTL];
754 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
756 val &= PL080_CONTROL_SWIDTH_MASK;
757 val >>= PL080_CONTROL_SWIDTH_SHIFT;
761 case PL080_WIDTH_8BIT:
763 case PL080_WIDTH_16BIT:
766 case PL080_WIDTH_32BIT:
773 /* The channel should be paused when calling this */
774 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
776 struct pl08x_driver_data *pl08x = plchan->host;
777 const u32 *llis_va, *llis_va_limit;
778 struct pl08x_phy_chan *ch;
780 struct pl08x_txd *txd;
785 ch = plchan->phychan;
792 * Follow the LLIs to get the number of remaining
793 * bytes in the currently active transaction.
795 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2;
797 /* First get the remaining bytes in the active transfer */
798 bytes = get_bytes_in_phy_channel(ch);
803 llis_va = txd->llis_va;
804 llis_bus = txd->llis_bus;
806 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
807 BUG_ON(clli < llis_bus || clli >= llis_bus +
808 sizeof(u32) * llis_max_words);
811 * Locate the next LLI - as this is an array,
812 * it's simple maths to find.
814 llis_va += (clli - llis_bus) / sizeof(u32);
816 llis_va_limit = llis_va + llis_max_words;
818 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
819 bytes += get_bytes_in_lli(ch, llis_va);
822 * A LLI pointer going backward terminates the LLI list
824 if (llis_va[PL080_LLI_LLI] <= clli)
832 * Allocate a physical channel for a virtual channel
834 * Try to locate a physical channel to be used for this transfer. If all
835 * are taken return NULL and the requester will have to cope by using
836 * some fallback PIO mode or retrying later.
838 static struct pl08x_phy_chan *
839 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
840 struct pl08x_dma_chan *virt_chan)
842 struct pl08x_phy_chan *ch = NULL;
846 //printk("this is debug virt_chan->id = %d %s %s %d\n",virt_chan->chan_id,__FILE__,__func__,__LINE__);
848 ch = &pl08x->phy_chans[virt_chan->chan_id];
850 spin_lock_irqsave(&ch->lock, flags);
852 if (!ch->locked && !ch->serving) {
853 ch->serving = virt_chan;
854 spin_unlock_irqrestore(&ch->lock, flags);
858 spin_unlock_irqrestore(&ch->lock, flags);
860 for (i = 0; i < pl08x->vd->channels; i++) {
861 ch = &pl08x->phy_chans[i];
863 spin_lock_irqsave(&ch->lock, flags);
865 if (!ch->locked && !ch->serving) {
866 ch->serving = virt_chan;
867 spin_unlock_irqrestore(&ch->lock, flags);
871 spin_unlock_irqrestore(&ch->lock, flags);
874 if (i == pl08x->vd->channels) {
875 /* No physical channel available, cope with it */
882 /* Mark the physical channel as free. Note, this write is atomic. */
883 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
884 struct pl08x_phy_chan *ch)
890 * Try to allocate a physical channel. When successful, assign it to
891 * this virtual channel, and initiate the next descriptor. The
892 * virtual channel lock must be held at this point.
894 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
896 struct pl08x_driver_data *pl08x = plchan->host;
897 struct pl08x_phy_chan *ch;
899 ch = pl08x_get_phy_channel(pl08x, plchan);
901 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
902 plchan->state = PL08X_CHAN_WAITING;
903 plchan->waiting_at = jiffies;
907 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
908 ch->id, plchan->name);
910 plchan->phychan = ch;
911 plchan->state = PL08X_CHAN_RUNNING;
912 pl08x_start_next_txd(plchan);
915 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
916 struct pl08x_dma_chan *plchan)
918 struct pl08x_driver_data *pl08x = plchan->host;
920 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
921 ch->id, plchan->name);
924 * We do this without taking the lock; we're really only concerned
925 * about whether this pointer is NULL or not, and we're guaranteed
926 * that this will only be called when it _already_ is non-NULL.
928 ch->serving = plchan;
929 plchan->phychan = ch;
930 plchan->state = PL08X_CHAN_RUNNING;
931 pl08x_start_next_txd(plchan);
935 * Free a physical DMA channel, potentially reallocating it to another
936 * virtual channel if we have any pending.
938 static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
940 struct pl08x_driver_data *pl08x = plchan->host;
941 struct pl08x_dma_chan *p, *next;
942 unsigned long waiting_at;
945 waiting_at = jiffies;
948 * Find a waiting virtual channel for the next transfer.
949 * To be fair, time when each channel reached waiting state is compared
950 * to select channel that is waiting for the longest time.
952 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
953 if (p->state == PL08X_CHAN_WAITING &&
954 p->waiting_at <= waiting_at) {
956 waiting_at = p->waiting_at;
959 if (!next && pl08x->has_slave) {
960 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
961 if (p->state == PL08X_CHAN_WAITING &&
962 p->waiting_at <= waiting_at) {
964 waiting_at = p->waiting_at;
968 /* Ensure that the physical channel is stopped */
969 pl08x_terminate_phy_chan(pl08x, plchan->phychan);
975 * Eww. We know this isn't going to deadlock
976 * but lockdep probably doesn't.
978 spin_lock(&next->vc.lock);
979 /* Re-check the state now that we have the lock */
980 success = next->state == PL08X_CHAN_WAITING;
982 pl08x_phy_reassign_start(plchan->phychan, next);
983 spin_unlock(&next->vc.lock);
985 /* If the state changed, try to find another channel */
989 /* No more jobs, so free up the physical channel */
990 pl08x_put_phy_channel(pl08x, plchan->phychan);
993 plchan->phychan = NULL;
994 plchan->state = PL08X_CHAN_IDLE;
1001 static inline unsigned int
1002 pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,
1008 if (pl08x->vd->ftdmac020) {
1010 val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
1011 FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1013 val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
1014 FTDMAC020_LLI_DST_WIDTH_SHIFT;
1017 val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>
1018 PL080_CONTROL_SWIDTH_SHIFT;
1020 val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>
1021 PL080_CONTROL_DWIDTH_SHIFT;
1025 case PL080_WIDTH_8BIT:
1027 case PL080_WIDTH_16BIT:
1029 case PL080_WIDTH_32BIT:
1038 static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
1040 u8 srcwidth, u8 dstwidth,
1046 * Remove all src, dst and transfer size bits, then set the
1047 * width and size according to the parameters. The bit offsets
1048 * are different in the FTDMAC020 so we need to accound for this.
1050 if (pl08x->vd->ftdmac020) {
1051 retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
1052 retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;
1053 retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;
1057 retbits |= PL080_WIDTH_8BIT <<
1058 FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1061 retbits |= PL080_WIDTH_16BIT <<
1062 FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1065 retbits |= PL080_WIDTH_32BIT <<
1066 FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1075 retbits |= PL080_WIDTH_8BIT <<
1076 FTDMAC020_LLI_DST_WIDTH_SHIFT;
1079 retbits |= PL080_WIDTH_16BIT <<
1080 FTDMAC020_LLI_DST_WIDTH_SHIFT;
1083 retbits |= PL080_WIDTH_32BIT <<
1084 FTDMAC020_LLI_DST_WIDTH_SHIFT;
1091 tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;
1092 retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;
1094 retbits &= ~PL080_CONTROL_DWIDTH_MASK;
1095 retbits &= ~PL080_CONTROL_SWIDTH_MASK;
1096 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
1100 retbits |= PL080_WIDTH_8BIT <<
1101 PL080_CONTROL_SWIDTH_SHIFT;
1104 retbits |= PL080_WIDTH_16BIT <<
1105 PL080_CONTROL_SWIDTH_SHIFT;
1108 retbits |= PL080_WIDTH_32BIT <<
1109 PL080_CONTROL_SWIDTH_SHIFT;
1118 retbits |= PL080_WIDTH_8BIT <<
1119 PL080_CONTROL_DWIDTH_SHIFT;
1122 retbits |= PL080_WIDTH_16BIT <<
1123 PL080_CONTROL_DWIDTH_SHIFT;
1126 retbits |= PL080_WIDTH_32BIT <<
1127 PL080_CONTROL_DWIDTH_SHIFT;
1134 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
1135 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
1141 struct pl08x_lli_build_data {
1142 struct pl08x_txd *txd;
1143 struct pl08x_bus_data srcbus;
1144 struct pl08x_bus_data dstbus;
1150 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
1151 * victim in case src & dest are not similarly aligned. i.e. If after aligning
1152 * masters address with width requirements of transfer (by sending few byte by
1153 * byte data), slave is still not aligned, then its width will be reduced to
1155 * - prefers the destination bus if both available
1156 * - prefers bus with fixed address (i.e. peripheral)
1158 static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,
1159 struct pl08x_lli_build_data *bd,
1160 struct pl08x_bus_data **mbus,
1161 struct pl08x_bus_data **sbus,
1168 * The FTDMAC020 only supports memory-to-memory transfer, so
1169 * source and destination always increase.
1171 if (pl08x->vd->ftdmac020) {
1175 dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);
1176 src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);
1180 * If either bus is not advancing, i.e. it is a peripheral, that
1181 * one becomes master
1184 *mbus = &bd->dstbus;
1185 *sbus = &bd->srcbus;
1186 } else if (!src_incr) {
1187 *mbus = &bd->srcbus;
1188 *sbus = &bd->dstbus;
1190 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
1191 *mbus = &bd->dstbus;
1192 *sbus = &bd->srcbus;
1194 *mbus = &bd->srcbus;
1195 *sbus = &bd->dstbus;
1201 * Fills in one LLI for a certain transfer descriptor and advance the counter
1203 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
1204 struct pl08x_lli_build_data *bd,
1205 int num_llis, int len, u32 cctl, u32 cctl2)
1207 u32 offset = num_llis * pl08x->lli_words;
1208 u32 *llis_va = bd->txd->llis_va + offset;
1209 dma_addr_t llis_bus = bd->txd->llis_bus;
1211 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
1214 printk("this is debug num_llis = %x lli_words = %x %s %s %d\n",
1215 num_llis,pl08x->lli_words,__FILE__,__func__,__LINE__);
1217 /* Advance the offset to next LLI. */
1218 offset += pl08x->lli_words;
1220 llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
1221 llis_va[PL080_LLI_DST] = bd->dstbus.addr;
1222 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
1223 llis_va[PL080_LLI_LLI] |= bd->lli_bus;
1224 llis_va[PL080_LLI_CCTL] = cctl;
1225 if (pl08x->vd->pl080s)
1226 llis_va[PL080S_LLI_CCTL2] = cctl2;
1228 if (pl08x->vd->ftdmac020) {
1229 /* FIXME: only memcpy so far so both increase */
1230 bd->srcbus.addr += len;
1231 bd->dstbus.addr += len;
1233 if (cctl & PL080_CONTROL_SRC_INCR)
1234 bd->srcbus.addr += len;
1235 if (cctl & PL080_CONTROL_DST_INCR)
1236 bd->dstbus.addr += len;
1239 BUG_ON(bd->remainder < len);
1241 bd->remainder -= len;
1244 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
1245 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
1246 int num_llis, size_t *total_bytes)
1248 *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len);
1249 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
1250 (*total_bytes) += len;
1254 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
1255 const u32 *llis_va, int num_llis)
1259 if (pl08x->vd->pl080s) {
1260 dev_vdbg(&pl08x->adev->dev,
1261 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
1262 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
1263 for (i = 0; i < num_llis; i++) {
1264 dev_vdbg(&pl08x->adev->dev,
1265 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1266 i, llis_va, llis_va[PL080_LLI_SRC],
1267 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
1268 llis_va[PL080_LLI_CCTL],
1269 llis_va[PL080S_LLI_CCTL2]);
1270 llis_va += pl08x->lli_words;
1273 //dev_vdbg(&pl08x->adev->dev,
1274 dev_info(&pl08x->adev->dev,
1275 "%-3s %-9s %-10s %-10s %-10s %s\n",
1276 "lli", "", "csrc", "cdst", "clli", "cctl");
1277 for (i = 0; i < num_llis; i++) {
1278 //dev_vdbg(&pl08x->adev->dev,
1279 dev_info(&pl08x->adev->dev,
1280 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1281 i, llis_va, llis_va[PL080_LLI_SRC],
1282 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
1283 llis_va[PL080_LLI_CCTL]);
1284 llis_va += pl08x->lli_words;
1289 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
1290 const u32 *llis_va, int num_llis) {}
1293 extern u64 dw_virt_to_phys(void *vaddr);
1295 * This fills in the table of LLIs for the transfer descriptor
1296 * Note that we assume we never have to change the burst sizes
1297 * Return 0 for error
1299 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
1300 struct pl08x_txd *txd)
1302 struct pl08x_bus_data *mbus, *sbus;
1303 struct pl08x_lli_build_data bd;
1305 u32 cctl, early_bytes = 0;
1306 size_t max_bytes_per_lli, total_bytes;
1307 u32 *llis_va, *last_lli;
1308 struct pl08x_sg *dsg;
1310 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
1311 if (!txd->llis_va) {
1312 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
1317 printk("this is debug txd->llis_bus = %llx pl08x->lli_buses = %x llis_va = %llx %s %s %d\n",
1318 txd->llis_bus,pl08x->lli_buses,dw_virt_to_phys(txd->llis_va),__FILE__,__func__,__LINE__);
1321 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
1324 /* Find maximum width of the source bus */
1325 bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true);
1327 /* Find maximum width of the destination bus */
1328 bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false);
1330 list_for_each_entry(dsg, &txd->dsg_list, node) {
1334 bd.srcbus.addr = dsg->src_addr;
1335 bd.dstbus.addr = dsg->dst_addr;
1336 bd.remainder = dsg->len;
1337 bd.srcbus.buswidth = bd.srcbus.maxwidth;
1338 bd.dstbus.buswidth = bd.dstbus.maxwidth;
1340 pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl);
1342 dev_vdbg(&pl08x->adev->dev,
1343 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
1344 (u64)bd.srcbus.addr,
1345 cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
1347 (u64)bd.dstbus.addr,
1348 cctl & PL080_CONTROL_DST_INCR ? "+" : "",
1351 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
1352 mbus == &bd.srcbus ? "src" : "dst",
1353 sbus == &bd.srcbus ? "src" : "dst");
1356 * Zero length is only allowed if all these requirements are
1358 * - flow controller is peripheral.
1359 * - src.addr is aligned to src.width
1360 * - dst.addr is aligned to dst.width
1362 * sg_len == 1 should be true, as there can be two cases here:
1364 * - Memory addresses are contiguous and are not scattered.
1365 * Here, Only one sg will be passed by user driver, with
1366 * memory address and zero length. We pass this to controller
1367 * and after the transfer it will receive the last burst
1368 * request from peripheral and so transfer finishes.
1370 * - Memory addresses are scattered and are not contiguous.
1371 * Here, Obviously as DMA controller doesn't know when a lli's
1372 * transfer gets over, it can't load next lli. So in this
1373 * case, there has to be an assumption that only one lli is
1374 * supported. Thus, we can't have scattered addresses.
1376 if (!bd.remainder) {
1379 /* FTDMAC020 only does memory-to-memory */
1380 if (pl08x->vd->ftdmac020)
1381 fc = PL080_FLOW_MEM2MEM;
1383 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
1384 PL080_CONFIG_FLOW_CONTROL_SHIFT;
1385 if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
1386 (fc <= PL080_FLOW_SRC2DST_SRC))) {
1387 dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
1392 if (!IS_BUS_ALIGNED(&bd.srcbus) ||
1393 !IS_BUS_ALIGNED(&bd.dstbus)) {
1394 dev_err(&pl08x->adev->dev,
1395 "%s src & dst address must be aligned to src"
1396 " & dst width if peripheral is flow controller",
1401 cctl = pl08x_lli_control_bits(pl08x, cctl,
1402 bd.srcbus.buswidth, bd.dstbus.buswidth,
1404 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
1410 * Send byte by byte for following cases
1411 * - Less than a bus width available
1412 * - until master bus is aligned
1414 if (bd.remainder < mbus->buswidth)
1415 early_bytes = bd.remainder;
1416 else if (!IS_BUS_ALIGNED(mbus)) {
1417 early_bytes = mbus->buswidth -
1418 (mbus->addr & (mbus->buswidth - 1));
1419 if ((bd.remainder - early_bytes) < mbus->buswidth)
1420 early_bytes = bd.remainder;
1424 dev_vdbg(&pl08x->adev->dev,
1425 "%s byte width LLIs (remain 0x%08zx)\n",
1426 __func__, bd.remainder);
1427 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
1428 num_llis++, &total_bytes);
1433 * Master now aligned
1434 * - if slave is not then we must set its width down
1436 if (!IS_BUS_ALIGNED(sbus)) {
1437 dev_dbg(&pl08x->adev->dev,
1438 "%s set down bus width to one byte\n",
1445 * Bytes transferred = tsize * src width, not
1448 max_bytes_per_lli = bd.srcbus.buswidth *
1449 pl08x->vd->max_transfer_size;
1450 dev_vdbg(&pl08x->adev->dev,
1451 "%s max bytes per lli = %zu\n",
1452 __func__, max_bytes_per_lli);
1455 * Make largest possible LLIs until less than one bus
1458 while (bd.remainder > (mbus->buswidth - 1)) {
1459 size_t lli_len, tsize, width;
1462 * If enough left try to send max possible,
1463 * otherwise try to send the remainder
1465 lli_len = min(bd.remainder, max_bytes_per_lli);
1468 * Check against maximum bus alignment:
1469 * Calculate actual transfer size in relation to
1470 * bus width an get a maximum remainder of the
1471 * highest bus width - 1
1473 width = max(mbus->buswidth, sbus->buswidth);
1474 lli_len = (lli_len / width) * width;
1475 tsize = lli_len / bd.srcbus.buswidth;
1477 dev_vdbg(&pl08x->adev->dev,
1478 "%s fill lli with single lli chunk of "
1479 "size 0x%08zx (remainder 0x%08zx)\n",
1480 __func__, lli_len, bd.remainder);
1482 cctl = pl08x_lli_control_bits(pl08x, cctl,
1483 bd.srcbus.buswidth, bd.dstbus.buswidth,
1485 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
1486 lli_len, cctl, tsize);
1487 total_bytes += lli_len;
1491 * Send any odd bytes
1494 dev_vdbg(&pl08x->adev->dev,
1495 "%s align with boundary, send odd bytes (remain %zu)\n",
1496 __func__, bd.remainder);
1497 prep_byte_width_lli(pl08x, &bd, &cctl,
1498 bd.remainder, num_llis++, &total_bytes);
1502 if (total_bytes != dsg->len) {
1503 dev_err(&pl08x->adev->dev,
1504 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
1505 __func__, total_bytes, dsg->len);
1509 if (num_llis >= MAX_NUM_TSFR_LLIS) {
1510 dev_err(&pl08x->adev->dev,
1511 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
1512 __func__, MAX_NUM_TSFR_LLIS);
1517 llis_va = txd->llis_va;
1518 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
1521 /* Link back to the first LLI. */
1522 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
1524 /* The final LLI terminates the LLI. */
1525 last_lli[PL080_LLI_LLI] = 0;
1526 /* The final LLI element shall also fire an interrupt. */
1527 if (pl08x->vd->ftdmac020)
1528 last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;
1530 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
1533 pl08x_dump_lli(pl08x, llis_va, num_llis);
1538 static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1539 struct pl08x_txd *txd)
1541 struct pl08x_sg *dsg, *_dsg;
1544 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
1546 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
1547 list_del(&dsg->node);
1554 static void pl08x_desc_free(struct virt_dma_desc *vd)
1556 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1557 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
1559 dma_descriptor_unmap(&vd->tx);
1561 pl08x_release_mux(plchan);
1563 pl08x_free_txd(plchan->host, txd);
1566 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1567 struct pl08x_dma_chan *plchan)
1571 vchan_get_all_descriptors(&plchan->vc, &head);
1572 vchan_dma_desc_free_list(&plchan->vc, &head);
1576 * The DMA ENGINE API
1578 static void pl08x_free_chan_resources(struct dma_chan *chan)
1580 /* Ensure all queued descriptors are freed */
1581 vchan_free_chan_resources(to_virt_chan(chan));
1584 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1585 struct dma_chan *chan, unsigned long flags)
1587 struct dma_async_tx_descriptor *retval = NULL;
1593 * Code accessing dma_async_is_complete() in a tight loop may give problems.
1594 * If slaves are relying on interrupts to signal completion this function
1595 * must not be called with interrupts disabled.
1597 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1598 dma_cookie_t cookie, struct dma_tx_state *txstate)
1600 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1601 struct virt_dma_desc *vd;
1602 unsigned long flags;
1603 enum dma_status ret;
1606 ret = dma_cookie_status(chan, cookie, txstate);
1607 if (ret == DMA_COMPLETE)
1611 * There's no point calculating the residue if there's
1612 * no txstate to store the value.
1615 if (plchan->state == PL08X_CHAN_PAUSED)
1620 spin_lock_irqsave(&plchan->vc.lock, flags);
1621 ret = dma_cookie_status(chan, cookie, txstate);
1622 if (ret != DMA_COMPLETE) {
1623 vd = vchan_find_desc(&plchan->vc, cookie);
1625 /* On the issued list, so hasn't been processed yet */
1626 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1627 struct pl08x_sg *dsg;
1629 list_for_each_entry(dsg, &txd->dsg_list, node)
1632 bytes = pl08x_getbytes_chan(plchan);
1635 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1638 * This cookie not complete yet
1639 * Get number of bytes left in the active transactions and queue
1641 dma_set_residue(txstate, bytes);
1643 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
1646 /* Whether waiting or running, we're in progress */
1650 /* PrimeCell DMA extension */
1651 struct burst_table {
1656 static const struct burst_table burst_sizes[] = {
1659 .reg = PL080_BSIZE_256,
1663 .reg = PL080_BSIZE_128,
1667 .reg = PL080_BSIZE_64,
1671 .reg = PL080_BSIZE_32,
1675 .reg = PL080_BSIZE_16,
1679 .reg = PL080_BSIZE_8,
1683 .reg = PL080_BSIZE_4,
1687 .reg = PL080_BSIZE_1,
1692 * Given the source and destination available bus masks, select which
1693 * will be routed to each port. We try to have source and destination
1694 * on separate ports, but always respect the allowable settings.
1696 static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)
1702 /* The FTDMAC020 use different bits to indicate src/dst bus */
1704 dst_ahb2 = FTDMAC020_LLI_DST_SEL;
1705 src_ahb2 = FTDMAC020_LLI_SRC_SEL;
1707 dst_ahb2 = PL080_CONTROL_DST_AHB2;
1708 src_ahb2 = PL080_CONTROL_SRC_AHB2;
1711 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1713 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1716 //printk("this is debug dst = %x src = %x cctl = %x %s %s %d\n",dst,src,cctl,__FILE__,__func__,__LINE__);
1721 static u32 pl08x_cctl(u32 cctl)
1723 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1724 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1725 PL080_CONTROL_PROT_MASK);
1727 /* Access the cell in privileged mode, non-bufferable, non-cacheable */
1728 return cctl | PL080_CONTROL_PROT_SYS;
1731 static u32 pl08x_width(enum dma_slave_buswidth width)
1734 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1735 return PL080_WIDTH_8BIT;
1736 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1737 return PL080_WIDTH_16BIT;
1738 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1739 return PL080_WIDTH_32BIT;
1745 static u32 pl08x_burst(u32 maxburst)
1749 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1750 if (burst_sizes[i].burstwords <= maxburst)
1753 return burst_sizes[i].reg;
1756 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1757 enum dma_slave_buswidth addr_width, u32 maxburst)
1759 u32 width, burst, cctl = 0;
1761 width = pl08x_width(addr_width);
1765 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1766 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1769 * If this channel will only request single transfers, set this
1770 * down to ONE element. Also select one element if no maxburst
1773 if (plchan->cd->single)
1776 burst = pl08x_burst(maxburst);
1777 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1778 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1780 return pl08x_cctl(cctl);
1784 * Slave transactions callback to the slave device to allow
1785 * synchronization of slave DMA signals with the DMAC enable
1787 static void pl08x_issue_pending(struct dma_chan *chan)
1789 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1790 unsigned long flags;
1792 plchan->chan_id = chan->chan_id;
1794 spin_lock_irqsave(&plchan->vc.lock, flags);
1795 if (vchan_issue_pending(&plchan->vc)) {
1796 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
1797 pl08x_phy_alloc_and_start(plchan);
1799 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1802 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
1804 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
1807 INIT_LIST_HEAD(&txd->dsg_list);
1811 static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)
1816 switch (pl08x->pd->memcpy_burst_size) {
1818 dev_err(&pl08x->adev->dev,
1819 "illegal burst size for memcpy, set to 1\n");
1821 case PL08X_BURST_SZ_1:
1822 cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
1823 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
1825 case PL08X_BURST_SZ_4:
1826 cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
1827 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
1829 case PL08X_BURST_SZ_8:
1830 cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
1831 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
1833 case PL08X_BURST_SZ_16:
1834 cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
1835 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
1837 case PL08X_BURST_SZ_32:
1838 cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
1839 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
1841 case PL08X_BURST_SZ_64:
1842 cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
1843 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
1845 case PL08X_BURST_SZ_128:
1846 cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
1847 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
1849 case PL08X_BURST_SZ_256:
1850 cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
1851 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
1855 switch (pl08x->pd->memcpy_bus_width) {
1857 dev_err(&pl08x->adev->dev,
1858 "illegal bus width for memcpy, set to 8 bits\n");
1860 case PL08X_BUS_WIDTH_8_BITS:
1861 cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
1862 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
1864 case PL08X_BUS_WIDTH_16_BITS:
1865 cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
1866 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
1868 case PL08X_BUS_WIDTH_32_BITS:
1869 cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
1870 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
1874 /* Protection flags */
1875 if (pl08x->pd->memcpy_prot_buff)
1876 cctl |= PL080_CONTROL_PROT_BUFF;
1877 if (pl08x->pd->memcpy_prot_cache)
1878 cctl |= PL080_CONTROL_PROT_CACHE;
1880 /* We are the kernel, so we are in privileged mode */
1881 cctl |= PL080_CONTROL_PROT_SYS;
1883 /* Both to be incremented or the code will break */
1884 cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1886 if (pl08x->vd->dualmaster)
1887 cctl |= pl08x_select_bus(false,
1894 static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)
1899 switch (pl08x->pd->memcpy_bus_width) {
1901 dev_err(&pl08x->adev->dev,
1902 "illegal bus width for memcpy, set to 8 bits\n");
1904 case PL08X_BUS_WIDTH_8_BITS:
1905 cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1906 PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1908 case PL08X_BUS_WIDTH_16_BITS:
1909 cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1910 PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1912 case PL08X_BUS_WIDTH_32_BITS:
1913 cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1914 PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1919 * By default mask the TC IRQ on all LLIs, it will be unmasked on
1920 * the last LLI item by other code.
1922 cctl |= FTDMAC020_LLI_TC_MSK;
1925 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL
1926 * and FTDMAC020_LLI_DSTAD_CTL as zero
1928 if (pl08x->vd->dualmaster)
1929 cctl |= pl08x_select_bus(true,
1937 * Initialize a descriptor to be used by memcpy submit
1939 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1940 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1941 size_t len, unsigned long flags)
1943 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1944 struct pl08x_driver_data *pl08x = plchan->host;
1945 struct pl08x_txd *txd;
1946 struct pl08x_sg *dsg;
1948 //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
1949 txd = pl08x_get_txd(plchan);
1951 dev_err(&pl08x->adev->dev,
1952 "%s no memory for descriptor\n", __func__);
1955 //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
1956 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1958 pl08x_free_txd(pl08x, txd);
1961 list_add_tail(&dsg->node, &txd->dsg_list);
1962 //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
1963 dsg->src_addr = src;
1964 dsg->dst_addr = dest;
1966 if (pl08x->vd->ftdmac020) {
1967 /* Writing CCFG zero ENABLES all interrupts */
1969 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);
1971 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1972 PL080_CONFIG_TC_IRQ_MASK |
1973 PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1974 txd->cctl = pl08x_memcpy_cctl(pl08x);
1976 //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
1977 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1979 pl08x_free_txd(pl08x, txd);
1982 //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
1983 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1986 static struct pl08x_txd *pl08x_init_txd(
1987 struct dma_chan *chan,
1988 enum dma_transfer_direction direction,
1989 dma_addr_t *slave_addr)
1991 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1992 struct pl08x_driver_data *pl08x = plchan->host;
1993 struct pl08x_txd *txd;
1994 enum dma_slave_buswidth addr_width;
1996 u8 src_buses, dst_buses;
1999 txd = pl08x_get_txd(plchan);
2001 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
2006 * Set up addresses, the PrimeCell configured address
2007 * will take precedence since this may configure the
2008 * channel target address dynamically at runtime.
2010 if (direction == DMA_MEM_TO_DEV) {
2011 //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2012 cctl = PL080_CONTROL_SRC_INCR;
2013 *slave_addr = plchan->cfg.dst_addr;
2014 addr_width = plchan->cfg.dst_addr_width;
2015 maxburst = plchan->cfg.dst_maxburst;
2016 src_buses = pl08x->mem_buses;
2017 dst_buses = plchan->cd->periph_buses;
2018 } else if (direction == DMA_DEV_TO_MEM) {
2019 //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2020 cctl = PL080_CONTROL_DST_INCR;
2021 *slave_addr = plchan->cfg.src_addr;
2022 addr_width = plchan->cfg.src_addr_width;
2023 maxburst = plchan->cfg.src_maxburst;
2024 src_buses = plchan->cd->periph_buses;
2025 dst_buses = pl08x->mem_buses;
2027 pl08x_free_txd(pl08x, txd);
2028 dev_err(&pl08x->adev->dev,
2029 "%s direction unsupported\n", __func__);
2033 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
2035 pl08x_free_txd(pl08x, txd);
2036 dev_err(&pl08x->adev->dev,
2037 "DMA slave configuration botched?\n");
2041 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses);
2043 if (plchan->cfg.device_fc)
2044 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
2045 PL080_FLOW_PER2MEM_PER;
2047 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
2050 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
2051 PL080_CONFIG_TC_IRQ_MASK |
2052 tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
2053 //printk("this is debug cctl = %x ccfg = %x %s %s %d\n",txd->cctl,txd->ccfg,__FILE__,__func__,__LINE__);
2055 ret = pl08x_request_mux(plchan);
2057 pl08x_free_txd(pl08x, txd);
2058 dev_dbg(&pl08x->adev->dev,
2059 "unable to mux for transfer on %s due to platform restrictions\n",
2064 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
2065 plchan->signal, plchan->name);
2067 /* Assign the flow control signal to this channel */
2068 if (direction == DMA_MEM_TO_DEV)
2069 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
2071 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
2076 static int pl08x_tx_add_sg(struct pl08x_txd *txd,
2077 enum dma_transfer_direction direction,
2078 dma_addr_t slave_addr,
2079 dma_addr_t buf_addr,
2082 struct pl08x_sg *dsg;
2084 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
2088 list_add_tail(&dsg->node, &txd->dsg_list);
2091 if (direction == DMA_MEM_TO_DEV) {
2092 dsg->src_addr = buf_addr;
2093 dsg->dst_addr = slave_addr;
2095 dsg->src_addr = slave_addr;
2096 dsg->dst_addr = buf_addr;
2102 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
2103 struct dma_chan *chan, struct scatterlist *sgl,
2104 unsigned int sg_len, enum dma_transfer_direction direction,
2105 unsigned long flags, void *context)
2107 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2108 struct pl08x_driver_data *pl08x = plchan->host;
2109 struct pl08x_txd *txd;
2110 struct scatterlist *sg;
2112 dma_addr_t slave_addr;
2114 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
2115 __func__, sg_dma_len(sgl), plchan->name);
2117 txd = pl08x_init_txd(chan, direction, &slave_addr);
2121 for_each_sg(sgl, sg, sg_len, tmp) {
2122 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
2126 printk("this is debug direction = %x slave_addr = %x addr = %x len = %x %s %s %d\n",
2127 direction,slave_addr,sg_dma_address(sg),sg_dma_len(sg),
2128 __FILE__,__func__,__LINE__);
2131 pl08x_release_mux(plchan);
2132 pl08x_free_txd(pl08x, txd);
2133 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
2139 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
2141 pl08x_release_mux(plchan);
2142 pl08x_free_txd(pl08x, txd);
2146 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
2149 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
2150 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
2151 size_t period_len, enum dma_transfer_direction direction,
2152 unsigned long flags)
2154 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2155 struct pl08x_driver_data *pl08x = plchan->host;
2156 struct pl08x_txd *txd;
2158 dma_addr_t slave_addr;
2160 dev_dbg(&pl08x->adev->dev,
2161 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
2162 __func__, period_len, buf_len,
2163 direction == DMA_MEM_TO_DEV ? "to" : "from",
2166 txd = pl08x_init_txd(chan, direction, &slave_addr);
2171 txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
2172 for (tmp = 0; tmp < buf_len; tmp += period_len) {
2173 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
2174 buf_addr + tmp, period_len);
2176 pl08x_release_mux(plchan);
2177 pl08x_free_txd(pl08x, txd);
2182 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
2184 pl08x_release_mux(plchan);
2185 pl08x_free_txd(pl08x, txd);
2189 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
2192 static int pl08x_config(struct dma_chan *chan,
2193 struct dma_slave_config *config)
2195 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2196 struct pl08x_driver_data *pl08x = plchan->host;
2201 /* Reject definitely invalid configurations */
2202 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
2203 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
2206 if (config->device_fc && pl08x->vd->pl080s) {
2207 dev_err(&pl08x->adev->dev,
2208 "%s: PL080S does not support peripheral flow control\n",
2214 printk("this is debug chan = %x chan-id = %d plchan = %x plchan->signal = %d %s %s %d\n",
2215 chan,chan->chan_id,plchan,plchan->signal,__FILE__,__func__,__LINE__);
2217 plchan->cfg = *config;
2222 static int pl08x_terminate_all(struct dma_chan *chan)
2224 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2225 struct pl08x_driver_data *pl08x = plchan->host;
2226 unsigned long flags;
2228 spin_lock_irqsave(&plchan->vc.lock, flags);
2229 if (!plchan->phychan && !plchan->at) {
2230 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2234 plchan->state = PL08X_CHAN_IDLE;
2236 if (plchan->phychan) {
2238 * Mark physical channel as free and free any slave
2241 pl08x_phy_free(plchan);
2243 /* Dequeue jobs and free LLIs */
2245 vchan_terminate_vdesc(&plchan->at->vd);
2248 /* Dequeue jobs not yet fired as well */
2249 pl08x_free_txd_list(pl08x, plchan);
2251 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2256 static void pl08x_synchronize(struct dma_chan *chan)
2258 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2260 printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
2261 vchan_synchronize(&plchan->vc);
2264 static int pl08x_pause(struct dma_chan *chan)
2266 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2267 unsigned long flags;
2270 * Anything succeeds on channels with no physical allocation and
2271 * no queued transfers.
2273 printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
2274 spin_lock_irqsave(&plchan->vc.lock, flags);
2275 if (!plchan->phychan && !plchan->at) {
2276 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2280 pl08x_pause_phy_chan(plchan->phychan);
2281 plchan->state = PL08X_CHAN_PAUSED;
2283 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2288 static int pl08x_resume(struct dma_chan *chan)
2290 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2291 unsigned long flags;
2294 * Anything succeeds on channels with no physical allocation and
2295 * no queued transfers.
2297 printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
2298 spin_lock_irqsave(&plchan->vc.lock, flags);
2299 if (!plchan->phychan && !plchan->at) {
2300 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2304 pl08x_resume_phy_chan(plchan->phychan);
2305 plchan->state = PL08X_CHAN_RUNNING;
2307 spin_unlock_irqrestore(&plchan->vc.lock, flags);
2312 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
2314 struct pl08x_dma_chan *plchan;
2315 char *name = chan_id;
2317 /* Reject channels for devices not bound to this driver */
2318 if (chan->device->dev->driver != &pl08x_amba_driver.drv)
2321 plchan = to_pl08x_chan(chan);
2323 /* Check that the channel is not taken! */
2324 if (!strcmp(plchan->name, name))
2329 EXPORT_SYMBOL_GPL(pl08x_filter_id);
2331 static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
2333 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2335 return plchan->cd == chan_id;
2339 * Just check that the device is there and active
2340 * TODO: turn this bit on/off depending on the number of physical channels
2341 * actually used, if it is zero... well shut it off. That will save some
2342 * power. Cut the clock at the same time.
2344 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
2346 /* The Nomadik variant does not have the config register */
2347 if (pl08x->vd->nomadik)
2349 /* The FTDMAC020 variant does this in another register */
2350 if (pl08x->vd->ftdmac020) {
2351 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR);
2354 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
2357 static irqreturn_t pl08x_irq(int irq, void *dev)
2359 struct pl08x_driver_data *pl08x = dev;
2360 u32 mask = 0, err, tc, i;
2362 /* check & clear - ERR & TC interrupts */
2363 err = readl(pl08x->base + PL080_ERR_STATUS);
2365 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
2367 writel(err, pl08x->base + PL080_ERR_CLEAR);
2369 tc = readl(pl08x->base + PL080_TC_STATUS);
2371 writel(tc, pl08x->base + PL080_TC_CLEAR);
2378 for (i = 0; i < pl08x->vd->channels; i++) {
2379 if ((BIT(i) & err) || (BIT(i) & tc)) {
2380 /* Locate physical channel */
2381 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
2382 struct pl08x_dma_chan *plchan = phychan->serving;
2383 struct pl08x_txd *tx;
2386 dev_err(&pl08x->adev->dev,
2387 "%s Error TC interrupt on unused channel: 0x%08x\n",
2392 spin_lock(&plchan->vc.lock);
2394 if (tx && tx->cyclic) {
2395 vchan_cyclic_callback(&tx->vd);
2399 * This descriptor is done, release its mux
2402 pl08x_release_mux(plchan);
2404 vchan_cookie_complete(&tx->vd);
2407 * And start the next descriptor (if any),
2408 * otherwise free this channel.
2410 if (vchan_next_desc(&plchan->vc))
2411 pl08x_start_next_txd(plchan);
2413 pl08x_phy_free(plchan);
2415 spin_unlock(&plchan->vc.lock);
2421 return mask ? IRQ_HANDLED : IRQ_NONE;
2424 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
2427 chan->name = chan->cd->bus_id;
2428 chan->cfg.src_addr = chan->cd->addr;
2429 chan->cfg.dst_addr = chan->cd->addr;
2433 * Initialise the DMAC memcpy/slave channels.
2434 * Make a local wrapper to hold required data
2436 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
2437 struct dma_device *dmadev, unsigned int channels, bool slave)
2439 struct pl08x_dma_chan *chan;
2442 INIT_LIST_HEAD(&dmadev->channels);
2445 * Register as many many memcpy as we have physical channels,
2446 * we won't always be able to use all but the code will have
2447 * to cope with that situation.
2449 for (i = 0; i < channels; i++) {
2450 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
2455 chan->state = PL08X_CHAN_IDLE;
2459 chan->cd = &pl08x->pd->slave_channels[i];
2461 * Some implementations have muxed signals, whereas some
2462 * use a mux in front of the signals and need dynamic
2463 * assignment of signals.
2466 pl08x_dma_slave_init(chan);
2468 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);
2473 chan->cd->bus_id = "memcpy";
2474 chan->cd->periph_buses = pl08x->pd->mem_buses;
2475 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
2482 dev_dbg(&pl08x->adev->dev,
2483 "initialize virtual channel \"%s\"\n",
2486 chan->vc.desc_free = pl08x_desc_free;
2487 vchan_init(&chan->vc, dmadev);
2489 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
2490 i, slave ? "slave" : "memcpy");
2494 static void pl08x_free_virtual_channels(struct dma_device *dmadev)
2496 struct pl08x_dma_chan *chan = NULL;
2497 struct pl08x_dma_chan *next;
2499 list_for_each_entry_safe(chan,
2500 next, &dmadev->channels, vc.chan.device_node) {
2501 list_del(&chan->vc.chan.device_node);
2506 #ifdef CONFIG_DEBUG_FS
2507 static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
2510 case PL08X_CHAN_IDLE:
2512 case PL08X_CHAN_RUNNING:
2514 case PL08X_CHAN_PAUSED:
2516 case PL08X_CHAN_WAITING:
2521 return "UNKNOWN STATE";
2524 static int pl08x_debugfs_show(struct seq_file *s, void *data)
2526 struct pl08x_driver_data *pl08x = s->private;
2527 struct pl08x_dma_chan *chan;
2528 struct pl08x_phy_chan *ch;
2529 unsigned long flags;
2532 seq_printf(s, "PL08x physical channels:\n");
2533 seq_printf(s, "CHANNEL:\tUSER:\n");
2534 seq_printf(s, "--------\t-----\n");
2535 for (i = 0; i < pl08x->vd->channels; i++) {
2536 struct pl08x_dma_chan *virt_chan;
2538 ch = &pl08x->phy_chans[i];
2540 spin_lock_irqsave(&ch->lock, flags);
2541 virt_chan = ch->serving;
2543 seq_printf(s, "%d\t\t%s%s\n",
2545 virt_chan ? virt_chan->name : "(none)",
2546 ch->locked ? " LOCKED" : "");
2548 spin_unlock_irqrestore(&ch->lock, flags);
2551 seq_printf(s, "\nPL08x virtual memcpy channels:\n");
2552 seq_printf(s, "CHANNEL:\tSTATE:\n");
2553 seq_printf(s, "--------\t------\n");
2554 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
2555 seq_printf(s, "%s\t\t%s\n", chan->name,
2556 pl08x_state_str(chan->state));
2559 if (pl08x->has_slave) {
2560 seq_printf(s, "\nPL08x virtual slave channels:\n");
2561 seq_printf(s, "CHANNEL:\tSTATE:\n");
2562 seq_printf(s, "--------\t------\n");
2563 list_for_each_entry(chan, &pl08x->slave.channels,
2564 vc.chan.device_node) {
2565 seq_printf(s, "%s\t\t%s\n", chan->name,
2566 pl08x_state_str(chan->state));
2573 DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
2575 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2577 /* Expose a simple debugfs interface to view all clocks */
2578 debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
2579 NULL, pl08x, &pl08x_debugfs_fops);
2583 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2589 static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
2592 struct pl08x_dma_chan *chan;
2594 /* Trying to get a slave channel from something with no slave support */
2595 if (!pl08x->has_slave)
2598 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
2599 if (chan->signal == id)
2600 return &chan->vc.chan;
2606 static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
2607 struct of_dma *ofdma)
2609 struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
2610 struct dma_chan *dma_chan;
2611 struct pl08x_dma_chan *plchan;
2616 if (dma_spec->args_count != 2) {
2617 dev_err(&pl08x->adev->dev,
2618 "DMA channel translation requires two cells\n");
2622 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
2624 dev_err(&pl08x->adev->dev,
2625 "DMA slave channel not found\n");
2629 plchan = to_pl08x_chan(dma_chan);
2630 dev_dbg(&pl08x->adev->dev,
2631 "translated channel for signal %d\n",
2634 /* Augment channel data for applicable AHB buses */
2635 plchan->cd->periph_buses = dma_spec->args[1];
2636 return dma_get_slave_channel(dma_chan);
2639 static int pl08x_of_probe(struct platform_device *adev,
2640 struct pl08x_driver_data *pl08x,
2641 struct device_node *np)
2643 struct pl08x_platform_data *pd;
2644 struct pl08x_channel_data *chanp = NULL;
2649 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
2653 /* Eligible bus masters for fetching LLIs */
2654 if (of_property_read_bool(np, "lli-bus-interface-ahb1"))
2655 pd->lli_buses |= PL08X_AHB1;
2656 if (of_property_read_bool(np, "lli-bus-interface-ahb2"))
2657 pd->lli_buses |= PL08X_AHB2;
2658 if (!pd->lli_buses) {
2659 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
2660 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
2663 /* Eligible bus masters for memory access */
2664 if (of_property_read_bool(np, "mem-bus-interface-ahb1"))
2665 pd->mem_buses |= PL08X_AHB1;
2666 if (of_property_read_bool(np, "mem-bus-interface-ahb2"))
2667 pd->mem_buses |= PL08X_AHB2;
2668 if (!pd->mem_buses) {
2669 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
2670 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
2673 /* Parse the memcpy channel properties */
2674 ret = of_property_read_u32(np, "memcpy-burst-size", &val);
2676 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
2681 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
2684 pd->memcpy_burst_size = PL08X_BURST_SZ_1;
2687 pd->memcpy_burst_size = PL08X_BURST_SZ_4;
2690 pd->memcpy_burst_size = PL08X_BURST_SZ_8;
2693 pd->memcpy_burst_size = PL08X_BURST_SZ_16;
2696 pd->memcpy_burst_size = PL08X_BURST_SZ_32;
2699 pd->memcpy_burst_size = PL08X_BURST_SZ_64;
2702 pd->memcpy_burst_size = PL08X_BURST_SZ_128;
2705 pd->memcpy_burst_size = PL08X_BURST_SZ_256;
2709 ret = of_property_read_u32(np, "memcpy-bus-width", &val);
2711 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
2716 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
2719 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;
2722 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;
2725 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;
2730 * Allocate channel data for all possible slave channels (one
2731 * for each possible signal), channels will then be allocated
2732 * for a device and have it's AHB interfaces set up at
2735 if (pl08x->vd->signals) {
2736 chanp = devm_kcalloc(&adev->dev,
2738 sizeof(struct pl08x_channel_data),
2743 pd->slave_channels = chanp;
2744 for (i = 0; i < pl08x->vd->signals; i++) {
2746 * chanp->periph_buses will be assigned at translation
2748 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);
2751 pd->num_slave_channels = pl08x->vd->signals;
2756 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
2760 static inline int pl08x_of_probe(struct platform_device *adev,
2761 struct pl08x_driver_data *pl08x,
2762 struct device_node *np)
2768 static int pl08x_probe(struct platform_device *adev) //, const struct amba_id *id)
2770 struct pl08x_driver_data *pl08x;
2771 struct vendor_data *vd;
2772 struct device_node *np = adev->dev.of_node;
2773 struct resource *res;
2778 //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2780 ret = amba_request_regions(adev, NULL);
2782 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2786 /* Ensure that we can do DMA */
2787 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2789 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2793 /* Create the driver state holder */
2794 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
2797 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2801 /* Assign useful pointers to the driver state */
2803 vd = of_device_get_match_data(&adev->dev);
2808 res = platform_get_resource_byname(adev, IORESOURCE_MEM, "sec_dma");
2809 pl08x->base = devm_ioremap_resource(&adev->dev, res);
2812 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2813 goto out_no_ioremap;
2816 if (vd->ftdmac020) {
2819 val = readl(pl08x->base + FTDMAC020_REVISION);
2820 dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",
2821 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
2822 val = readl(pl08x->base + FTDMAC020_FEATURE);
2823 dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d channels, "
2824 "%s built-in bridge, %s, %s linked lists\n",
2826 (val & BIT(10)) ? "no" : "has",
2827 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",
2828 (val & BIT(8)) ? "supports" : "does not support");
2830 /* Vendor data from feature register */
2831 if (!(val & BIT(8)))
2832 dev_warn(&pl08x->adev->dev,
2833 "linked lists not supported, required\n");
2834 vd->channels = (val >> 12) & 0x0f;
2835 vd->dualmaster = !!(val & BIT(9));
2838 /* Initialize memcpy engine */
2839 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
2840 pl08x->memcpy.dev = &adev->dev;
2841 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
2842 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2843 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2844 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
2845 pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
2846 pl08x->memcpy.device_config = pl08x_config;
2847 pl08x->memcpy.device_pause = pl08x_pause;
2848 pl08x->memcpy.device_resume = pl08x_resume;
2849 pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
2850 pl08x->memcpy.device_synchronize = pl08x_synchronize;
2851 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
2852 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
2853 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
2854 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
2856 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
2860 * Initialize slave engine, if the block has no signals, that means
2861 * we have no slave support.
2864 pl08x->has_slave = true;
2865 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
2866 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
2867 pl08x->slave.dev = &adev->dev;
2868 pl08x->slave.device_free_chan_resources =
2869 pl08x_free_chan_resources;
2870 pl08x->slave.device_prep_dma_interrupt =
2871 pl08x_prep_dma_interrupt;
2872 pl08x->slave.device_tx_status = pl08x_dma_tx_status;
2873 pl08x->slave.device_issue_pending = pl08x_issue_pending;
2874 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
2875 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
2876 pl08x->slave.device_config = pl08x_config;
2877 pl08x->slave.device_pause = pl08x_pause;
2878 pl08x->slave.device_resume = pl08x_resume;
2879 pl08x->slave.device_terminate_all = pl08x_terminate_all;
2880 pl08x->slave.device_synchronize = pl08x_synchronize;
2881 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
2882 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
2883 pl08x->slave.directions =
2884 BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2885 pl08x->slave.residue_granularity =
2886 DMA_RESIDUE_GRANULARITY_SEGMENT;
2889 /* Get the platform data */
2890 pl08x->pd = dev_get_platdata(&adev->dev);
2893 ret = pl08x_of_probe(adev, pl08x, np);
2895 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2896 goto out_no_platdata;
2899 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2900 dev_err(&adev->dev, "no platform data supplied\n");
2902 goto out_no_platdata;
2905 pl08x->slave.filter.map = pl08x->pd->slave_map;
2906 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
2907 pl08x->slave.filter.fn = pl08x_filter_fn;
2910 /* By default, AHB1 only. If dualmaster, from platform */
2911 pl08x->lli_buses = PL08X_AHB1;
2912 pl08x->mem_buses = PL08X_AHB1;
2913 if (pl08x->vd->dualmaster) {
2914 pl08x->lli_buses = pl08x->pd->lli_buses;
2915 pl08x->mem_buses = pl08x->pd->mem_buses;
2919 pl08x->lli_words = PL080S_LLI_WORDS;
2921 pl08x->lli_words = PL080_LLI_WORDS;
2922 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
2924 /* A DMA memory pool for LLIs, align on 1-byte boundary */
2925 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
2926 tsfr_size, PL08X_ALIGN, 0);
2928 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2930 goto out_no_lli_pool;
2933 /* Turn on the PL08x */
2934 pl08x_ensure_on(pl08x);
2936 /* Clear any pending interrupts */
2938 /* This variant has error IRQs in bits 16-19 */
2939 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR);
2941 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
2942 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2944 /* Attach the interrupt handler */
2945 irq = platform_get_irq(adev, 0);
2947 dev_err(&adev->dev, "Cannot get IRQ resource\n");
2951 ret = request_irq(irq, pl08x_irq, 0, DRIVER_NAME, pl08x);
2953 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2954 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2959 /* Initialize physical channels */
2960 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
2962 if (!pl08x->phy_chans) {
2963 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
2965 goto out_no_phychans;
2968 for (i = 0; i < vd->channels; i++) {
2969 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2972 ch->base = pl08x->base + PL080_Cx_BASE(i);
2973 if (vd->ftdmac020) {
2974 /* FTDMA020 has a special channel busy register */
2975 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;
2976 ch->reg_config = ch->base + FTDMAC020_CH_CFG;
2977 ch->reg_control = ch->base + FTDMAC020_CH_CSR;
2978 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;
2979 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;
2980 ch->reg_lli = ch->base + FTDMAC020_CH_LLP;
2981 ch->ftdmac020 = true;
2983 printk("this is debug i = %d ch->base = %x %s %s %d\n",i,ch->base,__FILE__,__func__,__LINE__);
2984 ch->reg_config = ch->base + vd->config_offset;
2985 ch->reg_control = ch->base + PL080_CH_CONTROL;
2986 ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
2987 ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
2988 ch->reg_lli = ch->base + PL080_CH_LLI;
2993 spin_lock_init(&ch->lock);
2996 * Nomadik variants can have channels that are locked
2997 * down for the secure world only. Lock up these channels
2998 * by perpetually serving a dummy virtual channel.
3003 val = readl(ch->reg_config);
3004 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
3005 dev_dbg(&adev->dev, "physical channel %d reserved for secure access only\n", i);
3010 //dev_dbg(&adev->dev, "physical channel %d is %s\n",
3011 dev_dbg(&adev->dev, "physical channel %d is %s\n",
3012 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
3015 /* Register as many memcpy channels as there are physical channels */
3016 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
3017 pl08x->vd->channels, false);
3019 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3020 dev_warn(&pl08x->adev->dev,
3021 "%s failed to enumerate memcpy channels - %d\n",
3026 /* Register slave channels */
3027 if (pl08x->has_slave) {
3028 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
3029 pl08x->pd->num_slave_channels, true);
3031 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3032 dev_warn(&pl08x->adev->dev,
3033 "%s failed to enumerate slave channels - %d\n",
3039 ret = dma_async_device_register(&pl08x->memcpy);
3041 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3042 dev_warn(&pl08x->adev->dev,
3043 "%s failed to register memcpy as an async device - %d\n",
3045 goto out_no_memcpy_reg;
3048 if (pl08x->has_slave) {
3049 ret = dma_async_device_register(&pl08x->slave);
3051 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3052 dev_warn(&pl08x->adev->dev,
3053 "%s failed to register slave as an async device - %d\n",
3055 goto out_no_slave_reg;
3059 platform_set_drvdata(adev, pl08x);
3060 init_pl08x_debugfs(pl08x);
3061 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3065 for (loop = 0xfe0;loop <= 0xffc; loop += 4) {
3066 dev_dbg(&pl08x->adev->dev, "periphid[0x%x] = %x ",
3067 loop,readl(pl08x->base + loop));
3070 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3071 dev_dbg(&pl08x->adev->dev, "DMA: PL080 at 0x%08llx irq %d\n",
3072 (unsigned long long)res->start, irq);
3077 dma_async_device_unregister(&pl08x->memcpy);
3079 if (pl08x->has_slave)
3080 pl08x_free_virtual_channels(&pl08x->slave);
3082 pl08x_free_virtual_channels(&pl08x->memcpy);
3084 kfree(pl08x->phy_chans);
3086 free_irq(irq, pl08x);
3088 dma_pool_destroy(pl08x->pool);
3091 iounmap(pl08x->base);
3095 //amba_release_regions(adev);
3096 printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
3100 /* PL080 has 8 channels and the PL080 have just 2 */
3101 static struct vendor_data vendor_pl080 = {
3102 .config_offset = PL080_CH_CONFIG,
3106 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3109 static struct vendor_data vendor_nomadik = {
3110 .config_offset = PL080_CH_CONFIG,
3115 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3118 static struct vendor_data vendor_pl080s = {
3119 .config_offset = PL080S_CH_CONFIG,
3123 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
3126 static struct vendor_data vendor_pl081 = {
3127 .config_offset = PL080_CH_CONFIG,
3130 .dualmaster = false,
3131 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3134 static struct vendor_data vendor_ftdmac020 = {
3135 .config_offset = PL080_CH_CONFIG,
3137 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3140 static const struct amba_id pl08x_ids[] = {
3141 /* Samsung PL080S variant */
3145 .data = &vendor_pl080s,
3151 .data = &vendor_pl080,
3157 .data = &vendor_pl081,
3159 /* Nomadik 8815 PL080 variant */
3163 .data = &vendor_nomadik,
3165 /* Faraday Technology FTDMAC020 */
3169 .data = &vendor_ftdmac020,
3174 MODULE_DEVICE_TABLE(amba, pl08x_ids);
3176 static struct amba_driver pl08x_amba_driver = {
3177 .drv.name = DRIVER_NAME,
3178 .id_table = pl08x_ids,
3179 .probe = pl08x_probe,
3182 static const struct of_device_id vic7110_dma_ids[] = {
3183 { .compatible = "starfive,pl080", .data = &vendor_pl080},
3186 MODULE_DEVICE_TABLE(of, vic7110_dma_ids);
3188 static struct platform_driver vic7110_pl08x_driver = {
3189 .probe = pl08x_probe,
3191 .name = DRIVER_NAME,
3192 .of_match_table = vic7110_dma_ids,
3196 module_platform_driver(vic7110_pl08x_driver);
3198 MODULE_LICENSE("GPL");
3199 MODULE_AUTHOR("Huan Feng <huan.feng@starfivetech.com>");
3200 MODULE_DESCRIPTION("Starfive VIC7110 CRYP DMA driver");