From 9b746045e4cd02cd08c482b433068a35a1e8a237 Mon Sep 17 00:00:00 2001 From: samin Date: Mon, 22 Nov 2021 14:14:37 +0800 Subject: [PATCH] dmaengine: dw-axi-dmac: Supports channels > 8 or hardware handshakes > 16. 1)Different num of chan/hw-handshakes have different register offsets. When the channel > 8, DMA uses DMAC_CHENREG2 instead of DMAC_CHENREG. When the channel > 8 or hw-handshake > 16, DMA uses CHx_CFG2 instead of CHx_CFG. 2)add OSR(Outstanding Request Limit) configuration OSR can greatly improve performance in dma-memcpy. Signed-off-by: samin --- drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c | 148 ++++++++++++++++++++----- drivers/dma/dw-axi-dmac/dw-axi-dmac.h | 58 +++++++++- 2 files changed, 177 insertions(+), 29 deletions(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 35993ab..50fb415 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -148,33 +148,79 @@ static inline u32 axi_chan_irq_read(struct axi_dma_chan *chan) return axi_chan_ioread32(chan, CH_INTSTATUS); } +static void axi_chan_set_multi_reg(struct axi_dma_chip *chip) +{ + struct dma_multi *multi = &chip->multi; + + /* cfg_2 Exists: (DMAX_NUM_CHANNELS > 8 || DMAX_NUM_HS_IF > 16) */ + if (multi->ch_cfg_2 == true) { + multi->cfg.ch_cfg_priority_pos = CH_CFG_H_PRIORITY_POS_2; + multi->cfg.ch_cfg_dst_per_pos = CH_CFG_L_DST_PER_POS_2; + multi->cfg.ch_cfg_src_per_pos = CH_CFG_L_SRC_PER_POS_2; + } else { + multi->cfg.ch_cfg_priority_pos = CH_CFG_H_PRIORITY_POS; + multi->cfg.ch_cfg_dst_per_pos = CH_CFG_H_DST_PER_POS; + multi->cfg.ch_cfg_src_per_pos = CH_CFG_H_SRC_PER_POS; + } + + /* en_2 Exists: DMAX_NUM_CHANNELS > 8 */ + if (multi->ch_enreg_2 == true) { + multi->en.ch_en = DMAC_CHEN_2; + multi->en.ch_en_shift = DMAC_CHAN_EN_SHIFT_2; + multi->en.ch_en_we_shift = DMAC_CHAN_EN_WE_SHIFT_2; + + multi->en.ch_susp = DMAC_CHSUSP_2; + multi->en.ch_susp_shift = DMAC_CHAN_SUSP_SHIFT_2; + multi->en.ch_susp_we_shift = DMAC_CHAN_SUSP_WE_SHIFT_2; + + multi->en.ch_abort = DMAC_CHABORT_2; + multi->en.ch_abort_shift = DMAC_CHAN_ABORT_SHIFT_2; + multi->en.ch_abort_we_shfit = DMAC_CHAN_ABORT_WE_SHIFT_2; + } else { + multi->en.ch_en = DMAC_CHEN; + multi->en.ch_en_shift = DMAC_CHAN_EN_SHIFT; + multi->en.ch_en_we_shift = DMAC_CHAN_EN_WE_SHIFT; + + multi->en.ch_susp = DMAC_CHSUSP; + multi->en.ch_susp_shift = DMAC_CHAN_SUSP_SHIFT; + multi->en.ch_susp_we_shift = DMAC_CHAN_SUSP_WE_SHIFT; + + multi->en.ch_abort = DMAC_CHABORT; + multi->en.ch_abort_shift = DMAC_CHAN_ABORT_SHIFT; + multi->en.ch_abort_we_shfit = DMAC_CHAN_ABORT_WE_SHIFT; + } +} + static inline void axi_chan_disable(struct axi_dma_chan *chan) { + struct dma_multi *multi = &chan->chip->multi; u32 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); - val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + val = axi_dma_ioread32(chan->chip, multi->en.ch_en); + val &= ~(BIT(chan->id) << multi->en.ch_en_shift); + val |= BIT(chan->id) << multi->en.ch_en_we_shift; + axi_dma_iowrite32(chan->chip, multi->en.ch_en, val); } static inline void axi_chan_enable(struct axi_dma_chan *chan) { + struct dma_multi *multi = &chan->chip->multi; u32 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | - BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + val = axi_dma_ioread32(chan->chip, multi->en.ch_en); + val |= BIT(chan->id) << multi->en.ch_en_shift | + BIT(chan->id) << multi->en.ch_en_we_shift; + axi_dma_iowrite32(chan->chip, multi->en.ch_en, val); } static inline bool axi_chan_is_hw_enable(struct axi_dma_chan *chan) { + struct dma_multi *multi = &chan->chip->multi; u32 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val = axi_dma_ioread32(chan->chip, multi->en.ch_en); - return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT)); + return !!(val & (BIT(chan->id) << multi->en.ch_en_shift)); } static void axi_dma_hw_init(struct axi_dma_chip *chip) @@ -335,8 +381,9 @@ static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) { + struct dma_multi *multi = &chan->chip->multi; u32 priority = chan->chip->dw->hdata->priority[chan->id]; - u32 reg, irq_mask; + u32 reg_lo, reg_hi, irq_mask; u8 lms = 0; /* Select AXI0 master for LLI fetching */ if (unlikely(axi_chan_is_hw_enable(chan))) { @@ -348,36 +395,65 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, axi_dma_enable(chan->chip); - reg = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS | + reg_lo = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS | DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_SRC_MULTBLK_TYPE_POS); - axi_chan_iowrite32(chan, CH_CFG_L, reg); - reg = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS | - priority << CH_CFG_H_PRIORITY_POS | + reg_hi = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS | + priority << multi->cfg.ch_cfg_priority_pos | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS); switch (chan->direction) { case DMA_MEM_TO_DEV: dw_axi_dma_set_byte_halfword(chan, true); - reg |= (chan->config.device_fc ? + reg_hi |= (chan->config.device_fc ? DWAXIDMAC_TT_FC_MEM_TO_PER_DST : DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC) << CH_CFG_H_TT_FC_POS; if (chan->chip->apb_regs) - reg |= (chan->id << CH_CFG_H_DST_PER_POS); + reg_hi |= (chan->id << CH_CFG_H_DST_PER_POS); break; case DMA_DEV_TO_MEM: - reg |= (chan->config.device_fc ? + reg_hi |= (chan->config.device_fc ? DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC) << CH_CFG_H_TT_FC_POS; if (chan->chip->apb_regs) - reg |= (chan->id << CH_CFG_H_SRC_PER_POS); + reg_hi |= (chan->id << CH_CFG_H_SRC_PER_POS); break; default: break; } - axi_chan_iowrite32(chan, CH_CFG_H, reg); + + reg_hi |= CH_CFG_H_MAX_OSR_LMT << CH_CFG_H_SRC_OSR_LMT_POS | + CH_CFG_H_MAX_OSR_LMT << CH_CFG_H_DST_OSR_LMT_POS; + + if (chan->hw_handshake_num) { + switch (chan->direction) { + case DMA_MEM_TO_DEV: + if (multi->ch_cfg_2 == true) + reg_lo |= chan->hw_handshake_num + << multi->cfg.ch_cfg_dst_per_pos; + else + reg_hi |= chan->hw_handshake_num + << multi->cfg.ch_cfg_dst_per_pos; + + break; + case DMA_DEV_TO_MEM: + if (multi->ch_cfg_2 == true) + reg_lo |= chan->hw_handshake_num + << multi->cfg.ch_cfg_src_per_pos; + else + reg_hi |= chan->hw_handshake_num + << multi->cfg.ch_cfg_src_per_pos; + + break; + default: + break; + } + } + + axi_chan_iowrite32(chan, CH_CFG_L, reg_lo); + axi_chan_iowrite32(chan, CH_CFG_H, reg_hi); write_chan_llp(chan, first->hw_desc[0].llp | lms); @@ -1114,16 +1190,17 @@ static int dma_chan_terminate_all(struct dma_chan *dchan) static int dma_chan_pause(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct dma_multi *multi = &chan->chip->multi; unsigned long flags; unsigned int timeout = 20; /* timeout iterations */ u32 val; spin_lock_irqsave(&chan->vc.lock, flags); - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT | - BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + val = axi_dma_ioread32(chan->chip, multi->en.ch_susp); + val |= BIT(chan->id) << multi->en.ch_susp_shift | + BIT(chan->id) << multi->en.ch_susp_we_shift; + axi_dma_iowrite32(chan->chip, multi->en.ch_susp, val); do { if (axi_chan_irq_read(chan) & DWAXIDMAC_IRQ_SUSPENDED) @@ -1144,12 +1221,13 @@ static int dma_chan_pause(struct dma_chan *dchan) /* Called in chan locked context */ static inline void axi_chan_resume(struct axi_dma_chan *chan) { + struct dma_multi *multi = &chan->chip->multi; u32 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT); - val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT); - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + val = axi_dma_ioread32(chan->chip, multi->en.ch_susp); + val &= ~(BIT(chan->id) << multi->en.ch_susp_shift); + val |= (BIT(chan->id) << multi->en.ch_susp_we_shift); + axi_dma_iowrite32(chan->chip, multi->en.ch_susp, val); chan->is_paused = false; } @@ -1293,6 +1371,20 @@ static int parse_device_properties(struct axi_dma_chip *chip) chip->dw->hdata->axi_rw_burst_len = tmp; } + + /* get number of handshak interface and configure multi reg */ + ret = device_property_read_u32(dev, "snps,num-hs-if", &tmp); + if (!ret) + chip->dw->hdata->nr_hs_if = tmp; + + if (chip->dw->hdata->nr_channels > 8) + chip->multi.ch_enreg_2 = true; + + if (chip->dw->hdata->nr_channels > 8 || chip->dw->hdata->nr_hs_if > 16) + chip->multi.ch_cfg_2 = true; + + axi_chan_set_multi_reg(chip); + return 0; } diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 380005a..17c5601 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -5,6 +5,7 @@ * Synopsys DesignWare AXI DMA Controller driver. * * Author: Eugeniy Paltsev + * Samin.guo */ #ifndef _AXI_DMA_PLATFORM_H @@ -18,13 +19,39 @@ #include "../virt-dma.h" -#define DMAC_MAX_CHANNELS 8 +#define DMAC_MAX_CHANNELS 16 #define DMAC_MAX_MASTERS 2 #define DMAC_MAX_BLK_SIZE 0x200000 +struct dma_ch_en { + u8 ch_en; + u8 ch_en_shift; + u8 ch_en_we_shift; + u8 ch_susp; + u8 ch_susp_shift; + u8 ch_susp_we_shift; + u8 ch_abort; + u8 ch_abort_shift; + u8 ch_abort_we_shfit; +}; + +struct dma_ch_cfg { + u8 ch_cfg_priority_pos; + u8 ch_cfg_dst_per_pos; + u8 ch_cfg_src_per_pos; +}; + +struct dma_multi { + bool ch_cfg_2; + bool ch_enreg_2; + struct dma_ch_cfg cfg; + struct dma_ch_en en; +}; + struct dw_axi_dma_hcfg { u32 nr_channels; u32 nr_masters; + u32 nr_hs_if; u32 m_data_width; u32 block_size[DMAC_MAX_CHANNELS]; u32 priority[DMAC_MAX_CHANNELS]; @@ -68,6 +95,7 @@ struct axi_dma_chip { struct clk *core_clk; struct clk *cfgr_clk; struct dw_axi_dma *dw; + struct dma_multi multi; }; /* LLI == Linked List Item */ @@ -139,6 +167,15 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define DMAC_CHEN 0x018 /* R/W DMAC Channel Enable */ #define DMAC_CHEN_L 0x018 /* R/W DMAC Channel Enable 00-31 */ #define DMAC_CHEN_H 0x01C /* R/W DMAC Channel Enable 32-63 */ +#define DMAC_CHSUSP 0x018 /* R/W DMAC Channel suspend */ +#define DMAC_CHABORT 0x018 /* R/W DMAC Channel Abort */ + +#define DMAC_CHEN_2 0x018 /* R/W DMAC Channel Enable */ +#define DMAC_CHEN_L_2 0x018 /* R/W DMAC Channel Enable */ +#define DMAC_CHEN_H_2 0x01C /* R/W DMAC Channel Enable */ +#define DMAC_CHSUSP_2 0x020 /* R/W DMAC Channel Suspend */ +#define DMAC_CHABORT_2 0x028 /* R/W DMAC Channel Abort */ + #define DMAC_INTSTATUS 0x030 /* R DMAC Interrupt Status */ #define DMAC_COMMON_INTCLEAR 0x038 /* W DMAC Interrupt Clear */ #define DMAC_COMMON_INTSTATUS_ENA 0x040 /* R DMAC Interrupt Status Enable */ @@ -201,6 +238,18 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define DMAC_CHAN_SUSP_SHIFT 16 #define DMAC_CHAN_SUSP_WE_SHIFT 24 +#define DMAC_CHAN_ABORT_SHIFT 32 +#define DMAC_CHAN_ABORT_WE_SHIFT 40 + +#define DMAC_CHAN_EN_SHIFT_2 0 +#define DMAC_CHAN_EN_WE_SHIFT_2 16 + +#define DMAC_CHAN_SUSP_SHIFT_2 0 +#define DMAC_CHAN_SUSP_WE_SHIFT_2 16 + +#define DMAC_CHAN_ABORT_SHIFT_2 0 +#define DMAC_CHAN_ABORT_WE_SHIFT_2 16 + /* CH_CTL_H */ #define CH_CTL_H_ARLEN_EN BIT(6) #define CH_CTL_H_ARLEN_POS 7 @@ -257,9 +306,13 @@ enum { #define CH_CTL_L_SRC_MAST BIT(0) /* CH_CFG_H */ +#define CH_CFG_H_DST_OSR_LMT_POS 27 +#define CH_CFG_H_SRC_OSR_LMT_POS 23 +#define CH_CFG_H_MAX_OSR_LMT 0xf #define CH_CFG_H_PRIORITY_POS 17 #define CH_CFG_H_DST_PER_POS 12 #define CH_CFG_H_SRC_PER_POS 7 +#define CH_CFG_H_PRIORITY_POS_2 15 #define CH_CFG_H_HS_SEL_DST_POS 4 #define CH_CFG_H_HS_SEL_SRC_POS 3 enum { @@ -282,6 +335,9 @@ enum { /* CH_CFG_L */ #define CH_CFG_L_DST_MULTBLK_TYPE_POS 2 #define CH_CFG_L_SRC_MULTBLK_TYPE_POS 0 + +#define CH_CFG_L_DST_PER_POS_2 11 +#define CH_CFG_L_SRC_PER_POS_2 4 enum { DWAXIDMAC_MBLK_TYPE_CONTIGUOUS = 0, DWAXIDMAC_MBLK_TYPE_RELOAD, -- 2.7.4