drivers/dma/owl-dma.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 //
   3 // Actions Semi Owl SoCs DMA driver
   4 //
   5 // Copyright (c) 2014 Actions Semi Inc.
   6 // Author: David Liu <liuwei@actions-semi.com>
   7 //
   8 // Copyright (c) 2018 Linaro Ltd.
   9 // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
  10
  11 #include <linux/bitops.h>
  12 #include <linux/clk.h>
  13 #include <linux/delay.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmapool.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/io.h>
  21 #include <linux/mm.h>
  22 #include <linux/module.h>
  23 #include <linux/of.h>
  24 #include <linux/of_dma.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include "virt-dma.h"
  28
  29 #define OWL_DMA_FRAME_MAX_LENGTH                0xfffff
  30
  31 /* Global DMA Controller Registers */
  32 #define OWL_DMA_IRQ_PD0                         0x00
  33 #define OWL_DMA_IRQ_PD1                         0x04
  34 #define OWL_DMA_IRQ_PD2                         0x08
  35 #define OWL_DMA_IRQ_PD3                         0x0C
  36 #define OWL_DMA_IRQ_EN0                         0x10
  37 #define OWL_DMA_IRQ_EN1                         0x14
  38 #define OWL_DMA_IRQ_EN2                         0x18
  39 #define OWL_DMA_IRQ_EN3                         0x1C
  40 #define OWL_DMA_SECURE_ACCESS_CTL               0x20
  41 #define OWL_DMA_NIC_QOS                         0x24
  42 #define OWL_DMA_DBGSEL                          0x28
  43 #define OWL_DMA_IDLE_STAT                       0x2C
  44
  45 /* Channel Registers */
  46 #define OWL_DMA_CHAN_BASE(i)                    (0x100 + (i) * 0x100)
  47 #define OWL_DMAX_MODE                           0x00
  48 #define OWL_DMAX_SOURCE                         0x04
  49 #define OWL_DMAX_DESTINATION                    0x08
  50 #define OWL_DMAX_FRAME_LEN                      0x0C
  51 #define OWL_DMAX_FRAME_CNT                      0x10
  52 #define OWL_DMAX_REMAIN_FRAME_CNT               0x14
  53 #define OWL_DMAX_REMAIN_CNT                     0x18
  54 #define OWL_DMAX_SOURCE_STRIDE                  0x1C
  55 #define OWL_DMAX_DESTINATION_STRIDE             0x20
  56 #define OWL_DMAX_START                          0x24
  57 #define OWL_DMAX_PAUSE                          0x28
  58 #define OWL_DMAX_CHAINED_CTL                    0x2C
  59 #define OWL_DMAX_CONSTANT                       0x30
  60 #define OWL_DMAX_LINKLIST_CTL                   0x34
  61 #define OWL_DMAX_NEXT_DESCRIPTOR                0x38
  62 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM         0x3C
  63 #define OWL_DMAX_INT_CTL                        0x40
  64 #define OWL_DMAX_INT_STATUS                     0x44
  65 #define OWL_DMAX_CURRENT_SOURCE_POINTER         0x48
  66 #define OWL_DMAX_CURRENT_DESTINATION_POINTER    0x4C
  67
  68 /* OWL_DMAX_MODE Bits */
  69 #define OWL_DMA_MODE_TS(x)                      (((x) & GENMASK(5, 0)) << 0)
  70 #define OWL_DMA_MODE_ST(x)                      (((x) & GENMASK(1, 0)) << 8)
  71 #define OWL_DMA_MODE_ST_DEV                     OWL_DMA_MODE_ST(0)
  72 #define OWL_DMA_MODE_ST_DCU                     OWL_DMA_MODE_ST(2)
  73 #define OWL_DMA_MODE_ST_SRAM                    OWL_DMA_MODE_ST(3)
  74 #define OWL_DMA_MODE_DT(x)                      (((x) & GENMASK(1, 0)) << 10)
  75 #define OWL_DMA_MODE_DT_DEV                     OWL_DMA_MODE_DT(0)
  76 #define OWL_DMA_MODE_DT_DCU                     OWL_DMA_MODE_DT(2)
  77 #define OWL_DMA_MODE_DT_SRAM                    OWL_DMA_MODE_DT(3)
  78 #define OWL_DMA_MODE_SAM(x)                     (((x) & GENMASK(1, 0)) << 16)
  79 #define OWL_DMA_MODE_SAM_CONST                  OWL_DMA_MODE_SAM(0)
  80 #define OWL_DMA_MODE_SAM_INC                    OWL_DMA_MODE_SAM(1)
  81 #define OWL_DMA_MODE_SAM_STRIDE                 OWL_DMA_MODE_SAM(2)
  82 #define OWL_DMA_MODE_DAM(x)                     (((x) & GENMASK(1, 0)) << 18)
  83 #define OWL_DMA_MODE_DAM_CONST                  OWL_DMA_MODE_DAM(0)
  84 #define OWL_DMA_MODE_DAM_INC                    OWL_DMA_MODE_DAM(1)
  85 #define OWL_DMA_MODE_DAM_STRIDE                 OWL_DMA_MODE_DAM(2)
  86 #define OWL_DMA_MODE_PW(x)                      (((x) & GENMASK(2, 0)) << 20)
  87 #define OWL_DMA_MODE_CB                         BIT(23)
  88 #define OWL_DMA_MODE_NDDBW(x)                   (((x) & 0x1) << 28)
  89 #define OWL_DMA_MODE_NDDBW_32BIT                OWL_DMA_MODE_NDDBW(0)
  90 #define OWL_DMA_MODE_NDDBW_8BIT                 OWL_DMA_MODE_NDDBW(1)
  91 #define OWL_DMA_MODE_CFE                        BIT(29)
  92 #define OWL_DMA_MODE_LME                        BIT(30)
  93 #define OWL_DMA_MODE_CME                        BIT(31)
  94
  95 /* OWL_DMAX_LINKLIST_CTL Bits */
  96 #define OWL_DMA_LLC_SAV(x)                      (((x) & GENMASK(1, 0)) << 8)
  97 #define OWL_DMA_LLC_SAV_INC                     OWL_DMA_LLC_SAV(0)
  98 #define OWL_DMA_LLC_SAV_LOAD_NEXT               OWL_DMA_LLC_SAV(1)
  99 #define OWL_DMA_LLC_SAV_LOAD_PREV               OWL_DMA_LLC_SAV(2)
 100 #define OWL_DMA_LLC_DAV(x)                      (((x) & GENMASK(1, 0)) << 10)
 101 #define OWL_DMA_LLC_DAV_INC                     OWL_DMA_LLC_DAV(0)
 102 #define OWL_DMA_LLC_DAV_LOAD_NEXT               OWL_DMA_LLC_DAV(1)
 103 #define OWL_DMA_LLC_DAV_LOAD_PREV               OWL_DMA_LLC_DAV(2)
 104 #define OWL_DMA_LLC_SUSPEND                     BIT(16)
 105
 106 /* OWL_DMAX_INT_CTL Bits */
 107 #define OWL_DMA_INTCTL_BLOCK                    BIT(0)
 108 #define OWL_DMA_INTCTL_SUPER_BLOCK              BIT(1)
 109 #define OWL_DMA_INTCTL_FRAME                    BIT(2)
 110 #define OWL_DMA_INTCTL_HALF_FRAME               BIT(3)
 111 #define OWL_DMA_INTCTL_LAST_FRAME               BIT(4)
 112
 113 /* OWL_DMAX_INT_STATUS Bits */
 114 #define OWL_DMA_INTSTAT_BLOCK                   BIT(0)
 115 #define OWL_DMA_INTSTAT_SUPER_BLOCK             BIT(1)
 116 #define OWL_DMA_INTSTAT_FRAME                   BIT(2)
 117 #define OWL_DMA_INTSTAT_HALF_FRAME              BIT(3)
 118 #define OWL_DMA_INTSTAT_LAST_FRAME              BIT(4)
 119
 120 /* Pack shift and newshift in a single word */
 121 #define BIT_FIELD(val, width, shift, newshift)  \
 122                 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
 123
 124 /* Frame count value is fixed as 1 */
 125 #define FCNT_VAL                                0x1
 126
 127 /**
 128  * enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link
 129  * list for dma transfer
 130  * @OWL_DMADESC_NEXT_LLI: physical address of the next link list
 131  * @OWL_DMADESC_SADDR: source physical address
 132  * @OWL_DMADESC_DADDR: destination physical address
 133  * @OWL_DMADESC_FLEN: frame length
 134  * @OWL_DMADESC_SRC_STRIDE: source stride
 135  * @OWL_DMADESC_DST_STRIDE: destination stride
 136  * @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config
 137  * @OWL_DMADESC_CTRLB: interrupt config
 138  * @OWL_DMADESC_CONST_NUM: data for constant fill
 139  * @OWL_DMADESC_SIZE: max size of this enum
 140  */
 141 enum owl_dmadesc_offsets {
 142         OWL_DMADESC_NEXT_LLI = 0,
 143         OWL_DMADESC_SADDR,
 144         OWL_DMADESC_DADDR,
 145         OWL_DMADESC_FLEN,
 146         OWL_DMADESC_SRC_STRIDE,
 147         OWL_DMADESC_DST_STRIDE,
 148         OWL_DMADESC_CTRLA,
 149         OWL_DMADESC_CTRLB,
 150         OWL_DMADESC_CONST_NUM,
 151         OWL_DMADESC_SIZE
 152 };
 153
 154 enum owl_dma_id {
 155         S900_DMA,
 156         S700_DMA,
 157 };
 158
 159 /**
 160  * struct owl_dma_lli - Link list for dma transfer
 161  * @hw: hardware link list
 162  * @phys: physical address of hardware link list
 163  * @node: node for txd's lli_list
 164  */
 165 struct owl_dma_lli {
 166         u32                     hw[OWL_DMADESC_SIZE];
 167         dma_addr_t              phys;
 168         struct list_head        node;
 169 };
 170
 171 /**
 172  * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
 173  * @vd: virtual DMA descriptor
 174  * @lli_list: link list of lli nodes
 175  * @cyclic: flag to indicate cyclic transfers
 176  */
 177 struct owl_dma_txd {
 178         struct virt_dma_desc    vd;
 179         struct list_head        lli_list;
 180         bool                    cyclic;
 181 };
 182
 183 /**
 184  * struct owl_dma_pchan - Holder for the physical channels
 185  * @id: physical index to this channel
 186  * @base: virtual memory base for the dma channel
 187  * @vchan: the virtual channel currently being served by this physical channel
 188  */
 189 struct owl_dma_pchan {
 190         u32                     id;
 191         void __iomem            *base;
 192         struct owl_dma_vchan    *vchan;
 193 };
 194
 195 /**
 196  * struct owl_dma_vchan - Wrapper for DMA ENGINE channel
 197  * @vc: wrapped virtual channel
 198  * @pchan: the physical channel utilized by this channel
 199  * @txd: active transaction on this channel
 200  * @cfg: slave configuration for this channel
 201  * @drq: physical DMA request ID for this channel
 202  */
 203 struct owl_dma_vchan {
 204         struct virt_dma_chan    vc;
 205         struct owl_dma_pchan    *pchan;
 206         struct owl_dma_txd      *txd;
 207         struct dma_slave_config cfg;
 208         u8                      drq;
 209 };
 210
 211 /**
 212  * struct owl_dma - Holder for the Owl DMA controller
 213  * @dma: dma engine for this instance
 214  * @base: virtual memory base for the DMA controller
 215  * @clk: clock for the DMA controller
 216  * @lock: a lock to use when change DMA controller global register
 217  * @lli_pool: a pool for the LLI descriptors
 218  * @irq: interrupt ID for the DMA controller
 219  * @nr_pchans: the number of physical channels
 220  * @pchans: array of data for the physical channels
 221  * @nr_vchans: the number of physical channels
 222  * @vchans: array of data for the physical channels
 223  * @devid: device id based on OWL SoC
 224  */
 225 struct owl_dma {
 226         struct dma_device       dma;
 227         void __iomem            *base;
 228         struct clk              *clk;
 229         spinlock_t              lock;
 230         struct dma_pool         *lli_pool;
 231         int                     irq;
 232
 233         unsigned int            nr_pchans;
 234         struct owl_dma_pchan    *pchans;
 235
 236         unsigned int            nr_vchans;
 237         struct owl_dma_vchan    *vchans;
 238         enum owl_dma_id         devid;
 239 };
 240
 241 static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
 242                          u32 val, bool state)
 243 {
 244         u32 regval;
 245
 246         regval = readl(pchan->base + reg);
 247
 248         if (state)
 249                 regval |= val;
 250         else
 251                 regval &= ~val;
 252
 253         writel(val, pchan->base + reg);
 254 }
 255
 256 static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
 257 {
 258         writel(data, pchan->base + reg);
 259 }
 260
 261 static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
 262 {
 263         return readl(pchan->base + reg);
 264 }
 265
 266 static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
 267 {
 268         u32 regval;
 269
 270         regval = readl(od->base + reg);
 271
 272         if (state)
 273                 regval |= val;
 274         else
 275                 regval &= ~val;
 276
 277         writel(val, od->base + reg);
 278 }
 279
 280 static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
 281 {
 282         writel(data, od->base + reg);
 283 }
 284
 285 static u32 dma_readl(struct owl_dma *od, u32 reg)
 286 {
 287         return readl(od->base + reg);
 288 }
 289
 290 static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
 291 {
 292         return container_of(dd, struct owl_dma, dma);
 293 }
 294
 295 static struct device *chan2dev(struct dma_chan *chan)
 296 {
 297         return &chan->dev->device;
 298 }
 299
 300 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
 301 {
 302         return container_of(chan, struct owl_dma_vchan, vc.chan);
 303 }
 304
 305 static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
 306 {
 307         return container_of(tx, struct owl_dma_txd, vd.tx);
 308 }
 309
 310 static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
 311 {
 312         u32 ctl;
 313
 314         ctl = BIT_FIELD(mode, 4, 28, 28) |
 315               BIT_FIELD(mode, 8, 16, 20) |
 316               BIT_FIELD(mode, 4, 8, 16) |
 317               BIT_FIELD(mode, 6, 0, 10) |
 318               BIT_FIELD(llc_ctl, 2, 10, 8) |
 319               BIT_FIELD(llc_ctl, 2, 8, 6);
 320
 321         return ctl;
 322 }
 323
 324 static inline u32 llc_hw_ctrlb(u32 int_ctl)
 325 {
 326         u32 ctl;
 327
 328         /*
 329          * Irrespective of the SoC, ctrlb value starts filling from
 330          * bit 18.
 331          */
 332         ctl = BIT_FIELD(int_ctl, 7, 0, 18);
 333
 334         return ctl;
 335 }
 336
 337 static u32 llc_hw_flen(struct owl_dma_lli *lli)
 338 {
 339         return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0);
 340 }
 341
 342 static void owl_dma_free_lli(struct owl_dma *od,
 343                              struct owl_dma_lli *lli)
 344 {
 345         list_del(&lli->node);
 346         dma_pool_free(od->lli_pool, lli, lli->phys);
 347 }
 348
 349 static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
 350 {
 351         struct owl_dma_lli *lli;
 352         dma_addr_t phys;
 353
 354         lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
 355         if (!lli)
 356                 return NULL;
 357
 358         INIT_LIST_HEAD(&lli->node);
 359         lli->phys = phys;
 360
 361         return lli;
 362 }
 363
 364 static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
 365                                            struct owl_dma_lli *prev,
 366                                            struct owl_dma_lli *next,
 367                                            bool is_cyclic)
 368 {
 369         if (!is_cyclic)
 370                 list_add_tail(&next->node, &txd->lli_list);
 371
 372         if (prev) {
 373                 prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys;
 374                 prev->hw[OWL_DMADESC_CTRLA] |=
 375                                         llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
 376         }
 377
 378         return next;
 379 }
 380
 381 static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
 382                                   struct owl_dma_lli *lli,
 383                                   dma_addr_t src, dma_addr_t dst,
 384                                   u32 len, enum dma_transfer_direction dir,
 385                                   struct dma_slave_config *sconfig,
 386                                   bool is_cyclic)
 387 {
 388         struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
 389         u32 mode, ctrlb;
 390
 391         mode = OWL_DMA_MODE_PW(0);
 392
 393         switch (dir) {
 394         case DMA_MEM_TO_MEM:
 395                 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
 396                         OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
 397                         OWL_DMA_MODE_DAM_INC;
 398
 399                 break;
 400         case DMA_MEM_TO_DEV:
 401                 mode |= OWL_DMA_MODE_TS(vchan->drq)
 402                         | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
 403                         | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
 404
 405                 /*
 406                  * Hardware only supports 32bit and 8bit buswidth. Since the
 407                  * default is 32bit, select 8bit only when requested.
 408                  */
 409                 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
 410                         mode |= OWL_DMA_MODE_NDDBW_8BIT;
 411
 412                 break;
 413         case DMA_DEV_TO_MEM:
 414                  mode |= OWL_DMA_MODE_TS(vchan->drq)
 415                         | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
 416                         | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
 417
 418                 /*
 419                  * Hardware only supports 32bit and 8bit buswidth. Since the
 420                  * default is 32bit, select 8bit only when requested.
 421                  */
 422                 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
 423                         mode |= OWL_DMA_MODE_NDDBW_8BIT;
 424
 425                 break;
 426         default:
 427                 return -EINVAL;
 428         }
 429
 430         lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode,
 431                                                   OWL_DMA_LLC_SAV_LOAD_NEXT |
 432                                                   OWL_DMA_LLC_DAV_LOAD_NEXT);
 433
 434         if (is_cyclic)
 435                 ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);
 436         else
 437                 ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
 438
 439         lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */
 440         lli->hw[OWL_DMADESC_SADDR] = src;
 441         lli->hw[OWL_DMADESC_DADDR] = dst;
 442         lli->hw[OWL_DMADESC_SRC_STRIDE] = 0;
 443         lli->hw[OWL_DMADESC_DST_STRIDE] = 0;
 444
 445         if (od->devid == S700_DMA) {
 446                 /* Max frame length is 1MB */
 447                 lli->hw[OWL_DMADESC_FLEN] = len;
 448                 /*
 449                  * On S700, word starts from offset 0x1C is shared between
 450                  * frame count and ctrlb, where first 12 bits are for frame
 451                  * count and rest of 20 bits are for ctrlb.
 452                  */
 453                 lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb;
 454         } else {
 455                 /*
 456                  * On S900, word starts from offset 0xC is shared between
 457                  * frame length (max frame length is 1MB) and frame count,
 458                  * where first 20 bits are for frame length and rest of
 459                  * 12 bits are for frame count.
 460                  */
 461                 lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20;
 462                 lli->hw[OWL_DMADESC_CTRLB] = ctrlb;
 463         }
 464
 465         return 0;
 466 }
 467
 468 static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
 469                                                struct owl_dma_vchan *vchan)
 470 {
 471         struct owl_dma_pchan *pchan = NULL;
 472         unsigned long flags;
 473         int i;
 474
 475         for (i = 0; i < od->nr_pchans; i++) {
 476                 pchan = &od->pchans[i];
 477
 478                 spin_lock_irqsave(&od->lock, flags);
 479                 if (!pchan->vchan) {
 480                         pchan->vchan = vchan;
 481                         spin_unlock_irqrestore(&od->lock, flags);
 482                         break;
 483                 }
 484
 485                 spin_unlock_irqrestore(&od->lock, flags);
 486         }
 487
 488         return pchan;
 489 }
 490
 491 static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
 492 {
 493         unsigned int val;
 494
 495         val = dma_readl(od, OWL_DMA_IDLE_STAT);
 496
 497         return !(val & (1 << pchan->id));
 498 }
 499
 500 static void owl_dma_terminate_pchan(struct owl_dma *od,
 501                                     struct owl_dma_pchan *pchan)
 502 {
 503         unsigned long flags;
 504         u32 irq_pd;
 505
 506         pchan_writel(pchan, OWL_DMAX_START, 0);
 507         pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 508
 509         spin_lock_irqsave(&od->lock, flags);
 510         dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
 511
 512         irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
 513         if (irq_pd & (1 << pchan->id)) {
 514                 dev_warn(od->dma.dev,
 515                          "terminating pchan %d that still has pending irq\n",
 516                          pchan->id);
 517                 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
 518         }
 519
 520         pchan->vchan = NULL;
 521
 522         spin_unlock_irqrestore(&od->lock, flags);
 523 }
 524
 525 static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
 526 {
 527         pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
 528 }
 529
 530 static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
 531 {
 532         pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
 533 }
 534
 535 static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
 536 {
 537         struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
 538         struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
 539         struct owl_dma_pchan *pchan = vchan->pchan;
 540         struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
 541         struct owl_dma_lli *lli;
 542         unsigned long flags;
 543         u32 int_ctl;
 544
 545         list_del(&vd->node);
 546
 547         vchan->txd = txd;
 548
 549         /* Wait for channel inactive */
 550         while (owl_dma_pchan_busy(od, pchan))
 551                 cpu_relax();
 552
 553         lli = list_first_entry(&txd->lli_list,
 554                                struct owl_dma_lli, node);
 555
 556         if (txd->cyclic)
 557                 int_ctl = OWL_DMA_INTCTL_BLOCK;
 558         else
 559                 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
 560
 561         pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
 562         pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
 563                      OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
 564         pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
 565         pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
 566
 567         /* Clear IRQ status for this pchan */
 568         pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 569
 570         spin_lock_irqsave(&od->lock, flags);
 571
 572         dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
 573
 574         spin_unlock_irqrestore(&od->lock, flags);
 575
 576         dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
 577
 578         /* Start DMA transfer for this pchan */
 579         pchan_writel(pchan, OWL_DMAX_START, 0x1);
 580
 581         return 0;
 582 }
 583
 584 static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
 585 {
 586         /* Ensure that the physical channel is stopped */
 587         owl_dma_terminate_pchan(od, vchan->pchan);
 588
 589         vchan->pchan = NULL;
 590 }
 591
 592 static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
 593 {
 594         struct owl_dma *od = dev_id;
 595         struct owl_dma_vchan *vchan;
 596         struct owl_dma_pchan *pchan;
 597         unsigned long pending;
 598         int i;
 599         unsigned int global_irq_pending, chan_irq_pending;
 600
 601         spin_lock(&od->lock);
 602
 603         pending = dma_readl(od, OWL_DMA_IRQ_PD0);
 604
 605         /* Clear IRQ status for each pchan */
 606         for_each_set_bit(i, &pending, od->nr_pchans) {
 607                 pchan = &od->pchans[i];
 608                 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 609         }
 610
 611         /* Clear pending IRQ */
 612         dma_writel(od, OWL_DMA_IRQ_PD0, pending);
 613
 614         /* Check missed pending IRQ */
 615         for (i = 0; i < od->nr_pchans; i++) {
 616                 pchan = &od->pchans[i];
 617                 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
 618                                    pchan_readl(pchan, OWL_DMAX_INT_STATUS);
 619
 620                 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
 621                 dma_readl(od, OWL_DMA_IRQ_PD0);
 622
 623                 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
 624
 625                 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
 626                         dev_dbg(od->dma.dev,
 627                                 "global and channel IRQ pending match err\n");
 628
 629                         /* Clear IRQ status for this pchan */
 630                         pchan_update(pchan, OWL_DMAX_INT_STATUS,
 631                                      0xff, false);
 632
 633                         /* Update global IRQ pending */
 634                         pending |= BIT(i);
 635                 }
 636         }
 637
 638         spin_unlock(&od->lock);
 639
 640         for_each_set_bit(i, &pending, od->nr_pchans) {
 641                 struct owl_dma_txd *txd;
 642
 643                 pchan = &od->pchans[i];
 644
 645                 vchan = pchan->vchan;
 646                 if (!vchan) {
 647                         dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
 648                                  pchan->id);
 649                         continue;
 650                 }
 651
 652                 spin_lock(&vchan->vc.lock);
 653
 654                 txd = vchan->txd;
 655                 if (txd) {
 656                         vchan->txd = NULL;
 657
 658                         vchan_cookie_complete(&txd->vd);
 659
 660                         /*
 661                          * Start the next descriptor (if any),
 662                          * otherwise free this channel.
 663                          */
 664                         if (vchan_next_desc(&vchan->vc))
 665                                 owl_dma_start_next_txd(vchan);
 666                         else
 667                                 owl_dma_phy_free(od, vchan);
 668                 }
 669
 670                 spin_unlock(&vchan->vc.lock);
 671         }
 672
 673         return IRQ_HANDLED;
 674 }
 675
 676 static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
 677 {
 678         struct owl_dma_lli *lli, *_lli;
 679
 680         if (unlikely(!txd))
 681                 return;
 682
 683         list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
 684                 owl_dma_free_lli(od, lli);
 685
 686         kfree(txd);
 687 }
 688
 689 static void owl_dma_desc_free(struct virt_dma_desc *vd)
 690 {
 691         struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
 692         struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
 693
 694         owl_dma_free_txd(od, txd);
 695 }
 696
 697 static int owl_dma_terminate_all(struct dma_chan *chan)
 698 {
 699         struct owl_dma *od = to_owl_dma(chan->device);
 700         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 701         unsigned long flags;
 702         LIST_HEAD(head);
 703
 704         spin_lock_irqsave(&vchan->vc.lock, flags);
 705
 706         if (vchan->pchan)
 707                 owl_dma_phy_free(od, vchan);
 708
 709         if (vchan->txd) {
 710                 owl_dma_desc_free(&vchan->txd->vd);
 711                 vchan->txd = NULL;
 712         }
 713
 714         vchan_get_all_descriptors(&vchan->vc, &head);
 715
 716         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 717
 718         vchan_dma_desc_free_list(&vchan->vc, &head);
 719
 720         return 0;
 721 }
 722
 723 static int owl_dma_config(struct dma_chan *chan,
 724                           struct dma_slave_config *config)
 725 {
 726         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 727
 728         /* Reject definitely invalid configurations */
 729         if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 730             config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 731                 return -EINVAL;
 732
 733         memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
 734
 735         return 0;
 736 }
 737
 738 static int owl_dma_pause(struct dma_chan *chan)
 739 {
 740         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 741         unsigned long flags;
 742
 743         spin_lock_irqsave(&vchan->vc.lock, flags);
 744
 745         owl_dma_pause_pchan(vchan->pchan);
 746
 747         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 748
 749         return 0;
 750 }
 751
 752 static int owl_dma_resume(struct dma_chan *chan)
 753 {
 754         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 755         unsigned long flags;
 756
 757         if (!vchan->pchan && !vchan->txd)
 758                 return 0;
 759
 760         dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
 761
 762         spin_lock_irqsave(&vchan->vc.lock, flags);
 763
 764         owl_dma_resume_pchan(vchan->pchan);
 765
 766         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 767
 768         return 0;
 769 }
 770
 771 static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
 772 {
 773         struct owl_dma_pchan *pchan;
 774         struct owl_dma_txd *txd;
 775         struct owl_dma_lli *lli;
 776         unsigned int next_lli_phy;
 777         size_t bytes;
 778
 779         pchan = vchan->pchan;
 780         txd = vchan->txd;
 781
 782         if (!pchan || !txd)
 783                 return 0;
 784
 785         /* Get remain count of current node in link list */
 786         bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
 787
 788         /* Loop through the preceding nodes to get total remaining bytes */
 789         if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
 790                 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
 791                 list_for_each_entry(lli, &txd->lli_list, node) {
 792                         /* Start from the next active node */
 793                         if (lli->phys == next_lli_phy) {
 794                                 list_for_each_entry(lli, &txd->lli_list, node)
 795                                         bytes += llc_hw_flen(lli);
 796                                 break;
 797                         }
 798                 }
 799         }
 800
 801         return bytes;
 802 }
 803
 804 static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
 805                                          dma_cookie_t cookie,
 806                                          struct dma_tx_state *state)
 807 {
 808         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 809         struct owl_dma_lli *lli;
 810         struct virt_dma_desc *vd;
 811         struct owl_dma_txd *txd;
 812         enum dma_status ret;
 813         unsigned long flags;
 814         size_t bytes = 0;
 815
 816         ret = dma_cookie_status(chan, cookie, state);
 817         if (ret == DMA_COMPLETE || !state)
 818                 return ret;
 819
 820         spin_lock_irqsave(&vchan->vc.lock, flags);
 821
 822         vd = vchan_find_desc(&vchan->vc, cookie);
 823         if (vd) {
 824                 txd = to_owl_txd(&vd->tx);
 825                 list_for_each_entry(lli, &txd->lli_list, node)
 826                         bytes += llc_hw_flen(lli);
 827         } else {
 828                 bytes = owl_dma_getbytes_chan(vchan);
 829         }
 830
 831         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 832
 833         dma_set_residue(state, bytes);
 834
 835         return ret;
 836 }
 837
 838 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
 839 {
 840         struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
 841         struct owl_dma_pchan *pchan;
 842
 843         pchan = owl_dma_get_pchan(od, vchan);
 844         if (!pchan)
 845                 return;
 846
 847         dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
 848
 849         vchan->pchan = pchan;
 850         owl_dma_start_next_txd(vchan);
 851 }
 852
 853 static void owl_dma_issue_pending(struct dma_chan *chan)
 854 {
 855         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 856         unsigned long flags;
 857
 858         spin_lock_irqsave(&vchan->vc.lock, flags);
 859         if (vchan_issue_pending(&vchan->vc)) {
 860                 if (!vchan->pchan)
 861                         owl_dma_phy_alloc_and_start(vchan);
 862         }
 863         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 864 }
 865
 866 static struct dma_async_tx_descriptor
 867                 *owl_dma_prep_memcpy(struct dma_chan *chan,
 868                                      dma_addr_t dst, dma_addr_t src,
 869                                      size_t len, unsigned long flags)
 870 {
 871         struct owl_dma *od = to_owl_dma(chan->device);
 872         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 873         struct owl_dma_txd *txd;
 874         struct owl_dma_lli *lli, *prev = NULL;
 875         size_t offset, bytes;
 876         int ret;
 877
 878         if (!len)
 879                 return NULL;
 880
 881         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 882         if (!txd)
 883                 return NULL;
 884
 885         INIT_LIST_HEAD(&txd->lli_list);
 886
 887         /* Process the transfer as frame by frame */
 888         for (offset = 0; offset < len; offset += bytes) {
 889                 lli = owl_dma_alloc_lli(od);
 890                 if (!lli) {
 891                         dev_warn(chan2dev(chan), "failed to allocate lli\n");
 892                         goto err_txd_free;
 893                 }
 894
 895                 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
 896
 897                 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
 898                                       bytes, DMA_MEM_TO_MEM,
 899                                       &vchan->cfg, txd->cyclic);
 900                 if (ret) {
 901                         dev_warn(chan2dev(chan), "failed to config lli\n");
 902                         goto err_txd_free;
 903                 }
 904
 905                 prev = owl_dma_add_lli(txd, prev, lli, false);
 906         }
 907
 908         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 909
 910 err_txd_free:
 911         owl_dma_free_txd(od, txd);
 912         return NULL;
 913 }
 914
 915 static struct dma_async_tx_descriptor
 916                 *owl_dma_prep_slave_sg(struct dma_chan *chan,
 917                                        struct scatterlist *sgl,
 918                                        unsigned int sg_len,
 919                                        enum dma_transfer_direction dir,
 920                                        unsigned long flags, void *context)
 921 {
 922         struct owl_dma *od = to_owl_dma(chan->device);
 923         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 924         struct dma_slave_config *sconfig = &vchan->cfg;
 925         struct owl_dma_txd *txd;
 926         struct owl_dma_lli *lli, *prev = NULL;
 927         struct scatterlist *sg;
 928         dma_addr_t addr, src = 0, dst = 0;
 929         size_t len;
 930         int ret, i;
 931
 932         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 933         if (!txd)
 934                 return NULL;
 935
 936         INIT_LIST_HEAD(&txd->lli_list);
 937
 938         for_each_sg(sgl, sg, sg_len, i) {
 939                 addr = sg_dma_address(sg);
 940                 len = sg_dma_len(sg);
 941
 942                 if (len > OWL_DMA_FRAME_MAX_LENGTH) {
 943                         dev_err(od->dma.dev,
 944                                 "frame length exceeds max supported length");
 945                         goto err_txd_free;
 946                 }
 947
 948                 lli = owl_dma_alloc_lli(od);
 949                 if (!lli) {
 950                         dev_err(chan2dev(chan), "failed to allocate lli");
 951                         goto err_txd_free;
 952                 }
 953
 954                 if (dir == DMA_MEM_TO_DEV) {
 955                         src = addr;
 956                         dst = sconfig->dst_addr;
 957                 } else {
 958                         src = sconfig->src_addr;
 959                         dst = addr;
 960                 }
 961
 962                 ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
 963                                       txd->cyclic);
 964                 if (ret) {
 965                         dev_warn(chan2dev(chan), "failed to config lli");
 966                         goto err_txd_free;
 967                 }
 968
 969                 prev = owl_dma_add_lli(txd, prev, lli, false);
 970         }
 971
 972         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 973
 974 err_txd_free:
 975         owl_dma_free_txd(od, txd);
 976
 977         return NULL;
 978 }
 979
 980 static struct dma_async_tx_descriptor
 981                 *owl_prep_dma_cyclic(struct dma_chan *chan,
 982                                      dma_addr_t buf_addr, size_t buf_len,
 983                                      size_t period_len,
 984                                      enum dma_transfer_direction dir,
 985                                      unsigned long flags)
 986 {
 987         struct owl_dma *od = to_owl_dma(chan->device);
 988         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 989         struct dma_slave_config *sconfig = &vchan->cfg;
 990         struct owl_dma_txd *txd;
 991         struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
 992         dma_addr_t src = 0, dst = 0;
 993         unsigned int periods = buf_len / period_len;
 994         int ret, i;
 995
 996         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 997         if (!txd)
 998                 return NULL;
 999
1000         INIT_LIST_HEAD(&txd->lli_list);
1001         txd->cyclic = true;
1002
1003         for (i = 0; i < periods; i++) {
1004                 lli = owl_dma_alloc_lli(od);
1005                 if (!lli) {
1006                         dev_warn(chan2dev(chan), "failed to allocate lli");
1007                         goto err_txd_free;
1008                 }
1009
1010                 if (dir == DMA_MEM_TO_DEV) {
1011                         src = buf_addr + (period_len * i);
1012                         dst = sconfig->dst_addr;
1013                 } else if (dir == DMA_DEV_TO_MEM) {
1014                         src = sconfig->src_addr;
1015                         dst = buf_addr + (period_len * i);
1016                 }
1017
1018                 ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
1019                                       dir, sconfig, txd->cyclic);
1020                 if (ret) {
1021                         dev_warn(chan2dev(chan), "failed to config lli");
1022                         goto err_txd_free;
1023                 }
1024
1025                 if (!first)
1026                         first = lli;
1027
1028                 prev = owl_dma_add_lli(txd, prev, lli, false);
1029         }
1030
1031         /* close the cyclic list */
1032         owl_dma_add_lli(txd, prev, first, true);
1033
1034         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
1035
1036 err_txd_free:
1037         owl_dma_free_txd(od, txd);
1038
1039         return NULL;
1040 }
1041
1042 static void owl_dma_free_chan_resources(struct dma_chan *chan)
1043 {
1044         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
1045
1046         /* Ensure all queued descriptors are freed */
1047         vchan_free_chan_resources(&vchan->vc);
1048 }
1049
1050 static inline void owl_dma_free(struct owl_dma *od)
1051 {
1052         struct owl_dma_vchan *vchan = NULL;
1053         struct owl_dma_vchan *next;
1054
1055         list_for_each_entry_safe(vchan,
1056                                  next, &od->dma.channels, vc.chan.device_node) {
1057                 list_del(&vchan->vc.chan.device_node);
1058                 tasklet_kill(&vchan->vc.task);
1059         }
1060 }
1061
1062 static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
1063                                          struct of_dma *ofdma)
1064 {
1065         struct owl_dma *od = ofdma->of_dma_data;
1066         struct owl_dma_vchan *vchan;
1067         struct dma_chan *chan;
1068         u8 drq = dma_spec->args[0];
1069
1070         if (drq > od->nr_vchans)
1071                 return NULL;
1072
1073         chan = dma_get_any_slave_channel(&od->dma);
1074         if (!chan)
1075                 return NULL;
1076
1077         vchan = to_owl_vchan(chan);
1078         vchan->drq = drq;
1079
1080         return chan;
1081 }
1082
1083 static const struct of_device_id owl_dma_match[] = {
1084         { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,},
1085         { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,},
1086         { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,},
1087         { /* sentinel */ },
1088 };
1089 MODULE_DEVICE_TABLE(of, owl_dma_match);
1090
1091 static int owl_dma_probe(struct platform_device *pdev)
1092 {
1093         struct device_node *np = pdev->dev.of_node;
1094         struct owl_dma *od;
1095         int ret, i, nr_channels, nr_requests;
1096
1097         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1098         if (!od)
1099                 return -ENOMEM;
1100
1101         od->base = devm_platform_ioremap_resource(pdev, 0);
1102         if (IS_ERR(od->base))
1103                 return PTR_ERR(od->base);
1104
1105         ret = of_property_read_u32(np, "dma-channels", &nr_channels);
1106         if (ret) {
1107                 dev_err(&pdev->dev, "can't get dma-channels\n");
1108                 return ret;
1109         }
1110
1111         ret = of_property_read_u32(np, "dma-requests", &nr_requests);
1112         if (ret) {
1113                 dev_err(&pdev->dev, "can't get dma-requests\n");
1114                 return ret;
1115         }
1116
1117         dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
1118                  nr_channels, nr_requests);
1119
1120         od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev);
1121
1122         od->nr_pchans = nr_channels;
1123         od->nr_vchans = nr_requests;
1124
1125         pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
1126
1127         platform_set_drvdata(pdev, od);
1128         spin_lock_init(&od->lock);
1129
1130         dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
1131         dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
1132         dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);
1133
1134         od->dma.dev = &pdev->dev;
1135         od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
1136         od->dma.device_tx_status = owl_dma_tx_status;
1137         od->dma.device_issue_pending = owl_dma_issue_pending;
1138         od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
1139         od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
1140         od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
1141         od->dma.device_config = owl_dma_config;
1142         od->dma.device_pause = owl_dma_pause;
1143         od->dma.device_resume = owl_dma_resume;
1144         od->dma.device_terminate_all = owl_dma_terminate_all;
1145         od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1146         od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1147         od->dma.directions = BIT(DMA_MEM_TO_MEM);
1148         od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1149
1150         INIT_LIST_HEAD(&od->dma.channels);
1151
1152         od->clk = devm_clk_get(&pdev->dev, NULL);
1153         if (IS_ERR(od->clk)) {
1154                 dev_err(&pdev->dev, "unable to get clock\n");
1155                 return PTR_ERR(od->clk);
1156         }
1157
1158         /*
1159          * Eventhough the DMA controller is capable of generating 4
1160          * IRQ's for DMA priority feature, we only use 1 IRQ for
1161          * simplification.
1162          */
1163         od->irq = platform_get_irq(pdev, 0);
1164         ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
1165                                dev_name(&pdev->dev), od);
1166         if (ret) {
1167                 dev_err(&pdev->dev, "unable to request IRQ\n");
1168                 return ret;
1169         }
1170
1171         /* Init physical channel */
1172         od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
1173                                   sizeof(struct owl_dma_pchan), GFP_KERNEL);
1174         if (!od->pchans)
1175                 return -ENOMEM;
1176
1177         for (i = 0; i < od->nr_pchans; i++) {
1178                 struct owl_dma_pchan *pchan = &od->pchans[i];
1179
1180                 pchan->id = i;
1181                 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
1182         }
1183
1184         /* Init virtual channel */
1185         od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
1186                                   sizeof(struct owl_dma_vchan), GFP_KERNEL);
1187         if (!od->vchans)
1188                 return -ENOMEM;
1189
1190         for (i = 0; i < od->nr_vchans; i++) {
1191                 struct owl_dma_vchan *vchan = &od->vchans[i];
1192
1193                 vchan->vc.desc_free = owl_dma_desc_free;
1194                 vchan_init(&vchan->vc, &od->dma);
1195         }
1196
1197         /* Create a pool of consistent memory blocks for hardware descriptors */
1198         od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
1199                                        sizeof(struct owl_dma_lli),
1200                                        __alignof__(struct owl_dma_lli),
1201                                        0);
1202         if (!od->lli_pool) {
1203                 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
1204                 return -ENOMEM;
1205         }
1206
1207         clk_prepare_enable(od->clk);
1208
1209         ret = dma_async_device_register(&od->dma);
1210         if (ret) {
1211                 dev_err(&pdev->dev, "failed to register DMA engine device\n");
1212                 goto err_pool_free;
1213         }
1214
1215         /* Device-tree DMA controller registration */
1216         ret = of_dma_controller_register(pdev->dev.of_node,
1217                                          owl_dma_of_xlate, od);
1218         if (ret) {
1219                 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1220                 goto err_dma_unregister;
1221         }
1222
1223         return 0;
1224
1225 err_dma_unregister:
1226         dma_async_device_unregister(&od->dma);
1227 err_pool_free:
1228         clk_disable_unprepare(od->clk);
1229         dma_pool_destroy(od->lli_pool);
1230
1231         return ret;
1232 }
1233
1234 static int owl_dma_remove(struct platform_device *pdev)
1235 {
1236         struct owl_dma *od = platform_get_drvdata(pdev);
1237
1238         of_dma_controller_free(pdev->dev.of_node);
1239         dma_async_device_unregister(&od->dma);
1240
1241         /* Mask all interrupts for this execution environment */
1242         dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
1243
1244         /* Make sure we won't have any further interrupts */
1245         devm_free_irq(od->dma.dev, od->irq, od);
1246
1247         owl_dma_free(od);
1248
1249         clk_disable_unprepare(od->clk);
1250         dma_pool_destroy(od->lli_pool);
1251
1252         return 0;
1253 }
1254
1255 static struct platform_driver owl_dma_driver = {
1256         .probe  = owl_dma_probe,
1257         .remove = owl_dma_remove,
1258         .driver = {
1259                 .name = "dma-owl",
1260                 .of_match_table = of_match_ptr(owl_dma_match),
1261         },
1262 };
1263
1264 static int owl_dma_init(void)
1265 {
1266         return platform_driver_register(&owl_dma_driver);
1267 }
1268 subsys_initcall(owl_dma_init);
1269
1270 static void __exit owl_dma_exit(void)
1271 {
1272         platform_driver_unregister(&owl_dma_driver);
1273 }
1274 module_exit(owl_dma_exit);
1275
1276 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1277 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1278 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1279 MODULE_LICENSE("GPL");