dmaengine: dw-axi-dmac: Supports channels > 8 or hardware handshakes > 16.

1)Different num of chan/hw-handshakes  have different register offsets.
When the channel > 8, DMA uses DMAC_CHENREG2 instead of DMAC_CHENREG.
When the channel > 8 or hw-handshake > 16, DMA uses CHx_CFG2 instead of CHx_CFG.

2)add OSR(Outstanding Request Limit) configuration
OSR can greatly improve performance in dma-memcpy.

Signed-off-by: samin <samin.guo@starfivetech.com>

diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index 35993ab9215473d618234765685d56fcf2e50b1c..50fb41544d0d6a4cde33233dcb225146bdaeaee3 100644 (file)
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -148,33 +148,79 @@ static inline u32 axi_chan_irq_read(struct axi_dma_chan *chan)
        return axi_chan_ioread32(chan, CH_INTSTATUS);
 }
 
+static void axi_chan_set_multi_reg(struct axi_dma_chip *chip)
+{
+       struct dma_multi *multi = &chip->multi;
+
+        /* cfg_2 Exists:  (DMAX_NUM_CHANNELS > 8 || DMAX_NUM_HS_IF > 16) */
+       if (multi->ch_cfg_2 == true) {
+               multi->cfg.ch_cfg_priority_pos = CH_CFG_H_PRIORITY_POS_2;
+               multi->cfg.ch_cfg_dst_per_pos = CH_CFG_L_DST_PER_POS_2;
+               multi->cfg.ch_cfg_src_per_pos = CH_CFG_L_SRC_PER_POS_2;
+       } else {
+               multi->cfg.ch_cfg_priority_pos = CH_CFG_H_PRIORITY_POS;
+               multi->cfg.ch_cfg_dst_per_pos = CH_CFG_H_DST_PER_POS;
+               multi->cfg.ch_cfg_src_per_pos = CH_CFG_H_SRC_PER_POS;
+       }
+
+       /* en_2 Exists: DMAX_NUM_CHANNELS > 8 */
+       if (multi->ch_enreg_2 == true) {
+               multi->en.ch_en = DMAC_CHEN_2;
+               multi->en.ch_en_shift = DMAC_CHAN_EN_SHIFT_2;
+               multi->en.ch_en_we_shift = DMAC_CHAN_EN_WE_SHIFT_2;
+
+               multi->en.ch_susp = DMAC_CHSUSP_2;
+               multi->en.ch_susp_shift = DMAC_CHAN_SUSP_SHIFT_2;
+               multi->en.ch_susp_we_shift = DMAC_CHAN_SUSP_WE_SHIFT_2;
+
+               multi->en.ch_abort = DMAC_CHABORT_2;
+               multi->en.ch_abort_shift = DMAC_CHAN_ABORT_SHIFT_2;
+               multi->en.ch_abort_we_shfit = DMAC_CHAN_ABORT_WE_SHIFT_2;
+       } else {
+               multi->en.ch_en = DMAC_CHEN;
+               multi->en.ch_en_shift = DMAC_CHAN_EN_SHIFT;
+               multi->en.ch_en_we_shift = DMAC_CHAN_EN_WE_SHIFT;
+
+               multi->en.ch_susp = DMAC_CHSUSP;
+               multi->en.ch_susp_shift = DMAC_CHAN_SUSP_SHIFT;
+               multi->en.ch_susp_we_shift = DMAC_CHAN_SUSP_WE_SHIFT;
+
+               multi->en.ch_abort = DMAC_CHABORT;
+               multi->en.ch_abort_shift = DMAC_CHAN_ABORT_SHIFT;
+               multi->en.ch_abort_we_shfit = DMAC_CHAN_ABORT_WE_SHIFT;
+       }
+}
+
 static inline void axi_chan_disable(struct axi_dma_chan *chan)
 {
+       struct dma_multi *multi = &chan->chip->multi;
        u32 val;
 
-       val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
-       val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT);
-       val |=   BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
-       axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
+       val = axi_dma_ioread32(chan->chip, multi->en.ch_en);
+       val &= ~(BIT(chan->id) << multi->en.ch_en_shift);
+       val |=   BIT(chan->id) << multi->en.ch_en_we_shift;
+       axi_dma_iowrite32(chan->chip, multi->en.ch_en, val);
 }
 
 static inline void axi_chan_enable(struct axi_dma_chan *chan)
 {
+       struct dma_multi *multi = &chan->chip->multi;
        u32 val;
 
-       val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
-       val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |
-              BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
-       axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
+       val = axi_dma_ioread32(chan->chip, multi->en.ch_en);
+       val |= BIT(chan->id) << multi->en.ch_en_shift |
+              BIT(chan->id) << multi->en.ch_en_we_shift;
+       axi_dma_iowrite32(chan->chip, multi->en.ch_en, val);
 }
 
 static inline bool axi_chan_is_hw_enable(struct axi_dma_chan *chan)
 {
+       struct dma_multi *multi = &chan->chip->multi;
        u32 val;
 
-       val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
+       val = axi_dma_ioread32(chan->chip, multi->en.ch_en);
 
-       return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT));
+       return !!(val & (BIT(chan->id) << multi->en.ch_en_shift));
 }
 
 static void axi_dma_hw_init(struct axi_dma_chip *chip)
@@ -335,8 +381,9 @@ static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set)
 static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
                                      struct axi_dma_desc *first)
 {
+       struct dma_multi *multi = &chan->chip->multi;
        u32 priority = chan->chip->dw->hdata->priority[chan->id];
-       u32 reg, irq_mask;
+       u32 reg_lo, reg_hi, irq_mask;
        u8 lms = 0; /* Select AXI0 master for LLI fetching */
 
        if (unlikely(axi_chan_is_hw_enable(chan))) {
@@ -348,36 +395,65 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
 
        axi_dma_enable(chan->chip);
 
-       reg = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS |
+       reg_lo = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS |
               DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_SRC_MULTBLK_TYPE_POS);
-       axi_chan_iowrite32(chan, CH_CFG_L, reg);
 
-       reg = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS |
-              priority << CH_CFG_H_PRIORITY_POS |
+       reg_hi = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS |
+              priority << multi->cfg.ch_cfg_priority_pos |
               DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS |
               DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS);
        switch (chan->direction) {
        case DMA_MEM_TO_DEV:
                dw_axi_dma_set_byte_halfword(chan, true);
-               reg |= (chan->config.device_fc ?
+               reg_hi |= (chan->config.device_fc ?
                        DWAXIDMAC_TT_FC_MEM_TO_PER_DST :
                        DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC)
                        << CH_CFG_H_TT_FC_POS;
                if (chan->chip->apb_regs)
-                       reg |= (chan->id << CH_CFG_H_DST_PER_POS);
+                       reg_hi |= (chan->id << CH_CFG_H_DST_PER_POS);
                break;
        case DMA_DEV_TO_MEM:
-               reg |= (chan->config.device_fc ?
+               reg_hi |= (chan->config.device_fc ?
                        DWAXIDMAC_TT_FC_PER_TO_MEM_SRC :
                        DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC)
                        << CH_CFG_H_TT_FC_POS;
                if (chan->chip->apb_regs)
-                       reg |= (chan->id << CH_CFG_H_SRC_PER_POS);
+                       reg_hi |= (chan->id << CH_CFG_H_SRC_PER_POS);
                break;
        default:
                break;
        }
-       axi_chan_iowrite32(chan, CH_CFG_H, reg);
+
+       reg_hi |= CH_CFG_H_MAX_OSR_LMT << CH_CFG_H_SRC_OSR_LMT_POS |
+               CH_CFG_H_MAX_OSR_LMT << CH_CFG_H_DST_OSR_LMT_POS;
+
+       if (chan->hw_handshake_num) {
+               switch (chan->direction) {
+               case DMA_MEM_TO_DEV:
+                       if (multi->ch_cfg_2 == true)
+                               reg_lo |= chan->hw_handshake_num
+                                       << multi->cfg.ch_cfg_dst_per_pos;
+                       else
+                               reg_hi |= chan->hw_handshake_num
+                                       << multi->cfg.ch_cfg_dst_per_pos;
+
+                       break;
+               case DMA_DEV_TO_MEM:
+                       if (multi->ch_cfg_2 == true)
+                               reg_lo |= chan->hw_handshake_num
+                                       << multi->cfg.ch_cfg_src_per_pos;
+                       else
+                               reg_hi |= chan->hw_handshake_num
+                                       << multi->cfg.ch_cfg_src_per_pos;
+
+                       break;
+               default:
+                       break;
+               }
+       }
+
+       axi_chan_iowrite32(chan, CH_CFG_L, reg_lo);
+       axi_chan_iowrite32(chan, CH_CFG_H, reg_hi);
 
        write_chan_llp(chan, first->hw_desc[0].llp | lms);
 
@@ -1114,16 +1190,17 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
 static int dma_chan_pause(struct dma_chan *dchan)
 {
        struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);
+       struct dma_multi *multi = &chan->chip->multi;
        unsigned long flags;
        unsigned int timeout = 20; /* timeout iterations */
        u32 val;
 
        spin_lock_irqsave(&chan->vc.lock, flags);
 
-       val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
-       val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT |
-              BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT;
-       axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
+       val = axi_dma_ioread32(chan->chip, multi->en.ch_susp);
+       val |= BIT(chan->id) << multi->en.ch_susp_shift |
+              BIT(chan->id) << multi->en.ch_susp_we_shift;
+       axi_dma_iowrite32(chan->chip, multi->en.ch_susp, val);
 
        do  {
                if (axi_chan_irq_read(chan) & DWAXIDMAC_IRQ_SUSPENDED)
@@ -1144,12 +1221,13 @@ static int dma_chan_pause(struct dma_chan *dchan)
 /* Called in chan locked context */
 static inline void axi_chan_resume(struct axi_dma_chan *chan)
 {
+       struct dma_multi *multi = &chan->chip->multi;
        u32 val;
 
-       val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
-       val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT);
-       val |=  (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT);
-       axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
+       val = axi_dma_ioread32(chan->chip, multi->en.ch_susp);
+       val &= ~(BIT(chan->id) << multi->en.ch_susp_shift);
+       val |=  (BIT(chan->id) << multi->en.ch_susp_we_shift);
+       axi_dma_iowrite32(chan->chip, multi->en.ch_susp, val);
 
        chan->is_paused = false;
 }
@@ -1293,6 +1371,20 @@ static int parse_device_properties(struct axi_dma_chip *chip)
                chip->dw->hdata->axi_rw_burst_len = tmp;
        }
 
+
+       /* get number of handshak interface and configure multi reg */
+       ret = device_property_read_u32(dev, "snps,num-hs-if", &tmp);
+       if (!ret)
+               chip->dw->hdata->nr_hs_if = tmp;
+
+       if (chip->dw->hdata->nr_channels > 8)
+               chip->multi.ch_enreg_2 = true;
+
+       if (chip->dw->hdata->nr_channels > 8 || chip->dw->hdata->nr_hs_if > 16)
+               chip->multi.ch_cfg_2 = true;
+
+       axi_chan_set_multi_reg(chip);
+
        return 0;
 }
 

diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
index 380005afde160e244d9f4260a6a7d85c1d53f888..17c5601db031967f8a60b7172b6e35a51c749df1 100644 (file)
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
@@ -5,6 +5,7 @@
  * Synopsys DesignWare AXI DMA Controller driver.
  *
  * Author: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
+ *         Samin.guo <samin.guo@starfivetech.com>
  */
 
 #ifndef _AXI_DMA_PLATFORM_H
@@ -18,13 +19,39 @@
 
 #include "../virt-dma.h"
 
-#define DMAC_MAX_CHANNELS      8
+#define DMAC_MAX_CHANNELS      16
 #define DMAC_MAX_MASTERS       2
 #define DMAC_MAX_BLK_SIZE      0x200000
 
+struct dma_ch_en {
+       u8 ch_en;
+       u8 ch_en_shift;
+       u8 ch_en_we_shift;
+       u8 ch_susp;
+       u8 ch_susp_shift;
+       u8 ch_susp_we_shift;
+       u8 ch_abort;
+       u8 ch_abort_shift;
+       u8 ch_abort_we_shfit;
+};
+
+struct dma_ch_cfg {
+       u8 ch_cfg_priority_pos;
+       u8 ch_cfg_dst_per_pos;
+       u8 ch_cfg_src_per_pos;
+};
+
+struct dma_multi {
+       bool ch_cfg_2;
+       bool ch_enreg_2;
+       struct dma_ch_cfg cfg;
+       struct dma_ch_en en;
+};
+
 struct dw_axi_dma_hcfg {
        u32     nr_channels;
        u32     nr_masters;
+       u32     nr_hs_if;
        u32     m_data_width;
        u32     block_size[DMAC_MAX_CHANNELS];
        u32     priority[DMAC_MAX_CHANNELS];
@@ -68,6 +95,7 @@ struct axi_dma_chip {
        struct clk              *core_clk;
        struct clk              *cfgr_clk;
        struct dw_axi_dma       *dw;
+       struct dma_multi        multi;
 };
 
 /* LLI == Linked List Item */
@@ -139,6 +167,15 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 #define DMAC_CHEN              0x018 /* R/W DMAC Channel Enable */
 #define DMAC_CHEN_L            0x018 /* R/W DMAC Channel Enable 00-31 */
 #define DMAC_CHEN_H            0x01C /* R/W DMAC Channel Enable 32-63 */
+#define DMAC_CHSUSP            0x018 /* R/W DMAC Channel suspend */
+#define DMAC_CHABORT           0x018 /* R/W DMAC Channel Abort */
+
+#define DMAC_CHEN_2            0x018 /* R/W DMAC Channel Enable */
+#define DMAC_CHEN_L_2          0x018 /* R/W DMAC Channel Enable */
+#define DMAC_CHEN_H_2          0x01C /* R/W DMAC Channel Enable */
+#define DMAC_CHSUSP_2          0x020 /* R/W DMAC Channel Suspend */
+#define DMAC_CHABORT_2         0x028 /* R/W DMAC Channel Abort */
+
 #define DMAC_INTSTATUS         0x030 /* R DMAC Interrupt Status */
 #define DMAC_COMMON_INTCLEAR   0x038 /* W DMAC Interrupt Clear */
 #define DMAC_COMMON_INTSTATUS_ENA 0x040 /* R DMAC Interrupt Status Enable */
@@ -201,6 +238,18 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 #define DMAC_CHAN_SUSP_SHIFT           16
 #define DMAC_CHAN_SUSP_WE_SHIFT                24
 
+#define DMAC_CHAN_ABORT_SHIFT          32
+#define DMAC_CHAN_ABORT_WE_SHIFT       40
+
+#define DMAC_CHAN_EN_SHIFT_2           0
+#define DMAC_CHAN_EN_WE_SHIFT_2                16
+
+#define DMAC_CHAN_SUSP_SHIFT_2         0
+#define DMAC_CHAN_SUSP_WE_SHIFT_2      16
+
+#define DMAC_CHAN_ABORT_SHIFT_2                0
+#define DMAC_CHAN_ABORT_WE_SHIFT_2     16
+
 /* CH_CTL_H */
 #define CH_CTL_H_ARLEN_EN              BIT(6)
 #define CH_CTL_H_ARLEN_POS             7
@@ -257,9 +306,13 @@ enum {
 #define CH_CTL_L_SRC_MAST              BIT(0)
 
 /* CH_CFG_H */
+#define CH_CFG_H_DST_OSR_LMT_POS       27
+#define CH_CFG_H_SRC_OSR_LMT_POS       23
+#define CH_CFG_H_MAX_OSR_LMT           0xf
 #define CH_CFG_H_PRIORITY_POS          17
 #define CH_CFG_H_DST_PER_POS           12
 #define CH_CFG_H_SRC_PER_POS           7
+#define CH_CFG_H_PRIORITY_POS_2                15
 #define CH_CFG_H_HS_SEL_DST_POS                4
 #define CH_CFG_H_HS_SEL_SRC_POS                3
 enum {
@@ -282,6 +335,9 @@ enum {
 /* CH_CFG_L */
 #define CH_CFG_L_DST_MULTBLK_TYPE_POS  2
 #define CH_CFG_L_SRC_MULTBLK_TYPE_POS  0
+
+#define CH_CFG_L_DST_PER_POS_2         11
+#define CH_CFG_L_SRC_PER_POS_2         4
 enum {
        DWAXIDMAC_MBLK_TYPE_CONTIGUOUS  = 0,
        DWAXIDMAC_MBLK_TYPE_RELOAD,