drivers/dma/sun6i-dma.c

   1 /*
   2  * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd
   3  * Author: Sugar <shuge@allwinnertech.com>
   4  *
   5  * Copyright (C) 2014 Maxime Ripard
   6  * Maxime Ripard <maxime.ripard@free-electrons.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/delay.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/dmapool.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/module.h>
  20 #include <linux/of_dma.h>
  21 #include <linux/of_device.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/reset.h>
  24 #include <linux/slab.h>
  25 #include <linux/types.h>
  26
  27 #include "virt-dma.h"
  28
  29 /*
  30  * Common registers
  31  */
  32 #define DMA_IRQ_EN(x)           ((x) * 0x04)
  33 #define DMA_IRQ_HALF                    BIT(0)
  34 #define DMA_IRQ_PKG                     BIT(1)
  35 #define DMA_IRQ_QUEUE                   BIT(2)
  36
  37 #define DMA_IRQ_CHAN_NR                 8
  38 #define DMA_IRQ_CHAN_WIDTH              4
  39
  40
  41 #define DMA_IRQ_STAT(x)         ((x) * 0x04 + 0x10)
  42
  43 #define DMA_STAT                0x30
  44
  45 /*
  46  * sun8i specific registers
  47  */
  48 #define SUN8I_DMA_GATE          0x20
  49 #define SUN8I_DMA_GATE_ENABLE   0x4
  50
  51 /*
  52  * Channels specific registers
  53  */
  54 #define DMA_CHAN_ENABLE         0x00
  55 #define DMA_CHAN_ENABLE_START           BIT(0)
  56 #define DMA_CHAN_ENABLE_STOP            0
  57
  58 #define DMA_CHAN_PAUSE          0x04
  59 #define DMA_CHAN_PAUSE_PAUSE            BIT(1)
  60 #define DMA_CHAN_PAUSE_RESUME           0
  61
  62 #define DMA_CHAN_LLI_ADDR       0x08
  63
  64 #define DMA_CHAN_CUR_CFG        0x0c
  65 #define DMA_CHAN_CFG_SRC_DRQ(x)         ((x) & 0x1f)
  66 #define DMA_CHAN_CFG_SRC_IO_MODE        BIT(5)
  67 #define DMA_CHAN_CFG_SRC_LINEAR_MODE    (0 << 5)
  68 #define DMA_CHAN_CFG_SRC_BURST(x)       (((x) & 0x3) << 7)
  69 #define DMA_CHAN_CFG_SRC_WIDTH(x)       (((x) & 0x3) << 9)
  70
  71 #define DMA_CHAN_CFG_DST_DRQ(x)         (DMA_CHAN_CFG_SRC_DRQ(x) << 16)
  72 #define DMA_CHAN_CFG_DST_IO_MODE        (DMA_CHAN_CFG_SRC_IO_MODE << 16)
  73 #define DMA_CHAN_CFG_DST_LINEAR_MODE    (DMA_CHAN_CFG_SRC_LINEAR_MODE << 16)
  74 #define DMA_CHAN_CFG_DST_BURST(x)       (DMA_CHAN_CFG_SRC_BURST(x) << 16)
  75 #define DMA_CHAN_CFG_DST_WIDTH(x)       (DMA_CHAN_CFG_SRC_WIDTH(x) << 16)
  76
  77 #define DMA_CHAN_CUR_SRC        0x10
  78
  79 #define DMA_CHAN_CUR_DST        0x14
  80
  81 #define DMA_CHAN_CUR_CNT        0x18
  82
  83 #define DMA_CHAN_CUR_PARA       0x1c
  84
  85
  86 /*
  87  * Various hardware related defines
  88  */
  89 #define LLI_LAST_ITEM   0xfffff800
  90 #define NORMAL_WAIT     8
  91 #define DRQ_SDRAM       1
  92
  93 /*
  94  * Hardware channels / ports representation
  95  *
  96  * The hardware is used in several SoCs, with differing numbers
  97  * of channels and endpoints. This structure ties those numbers
  98  * to a certain compatible string.
  99  */
 100 struct sun6i_dma_config {
 101         u32 nr_max_channels;
 102         u32 nr_max_requests;
 103         u32 nr_max_vchans;
 104         /*
 105          * In the datasheets/user manuals of newer Allwinner SoCs, a special
 106          * bit (bit 2 at register 0x20) is present.
 107          * It's named "DMA MCLK interface circuit auto gating bit" in the
 108          * documents, and the footnote of this register says that this bit
 109          * should be set up when initializing the DMA controller.
 110          * Allwinner A23/A33 user manuals do not have this bit documented,
 111          * however these SoCs really have and need this bit, as seen in the
 112          * BSP kernel source code.
 113          */
 114         bool gate_needed;
 115 };
 116
 117 /*
 118  * Hardware representation of the LLI
 119  *
 120  * The hardware will be fed the physical address of this structure,
 121  * and read its content in order to start the transfer.
 122  */
 123 struct sun6i_dma_lli {
 124         u32                     cfg;
 125         u32                     src;
 126         u32                     dst;
 127         u32                     len;
 128         u32                     para;
 129         u32                     p_lli_next;
 130
 131         /*
 132          * This field is not used by the DMA controller, but will be
 133          * used by the CPU to go through the list (mostly for dumping
 134          * or freeing it).
 135          */
 136         struct sun6i_dma_lli    *v_lli_next;
 137 };
 138
 139
 140 struct sun6i_desc {
 141         struct virt_dma_desc    vd;
 142         dma_addr_t              p_lli;
 143         struct sun6i_dma_lli    *v_lli;
 144 };
 145
 146 struct sun6i_pchan {
 147         u32                     idx;
 148         void __iomem            *base;
 149         struct sun6i_vchan      *vchan;
 150         struct sun6i_desc       *desc;
 151         struct sun6i_desc       *done;
 152 };
 153
 154 struct sun6i_vchan {
 155         struct virt_dma_chan    vc;
 156         struct list_head        node;
 157         struct dma_slave_config cfg;
 158         struct sun6i_pchan      *phy;
 159         u8                      port;
 160         u8                      irq_type;
 161         bool                    cyclic;
 162 };
 163
 164 struct sun6i_dma_dev {
 165         struct dma_device       slave;
 166         void __iomem            *base;
 167         struct clk              *clk;
 168         int                     irq;
 169         spinlock_t              lock;
 170         struct reset_control    *rstc;
 171         struct tasklet_struct   task;
 172         atomic_t                tasklet_shutdown;
 173         struct list_head        pending;
 174         struct dma_pool         *pool;
 175         struct sun6i_pchan      *pchans;
 176         struct sun6i_vchan      *vchans;
 177         const struct sun6i_dma_config *cfg;
 178 };
 179
 180 static struct device *chan2dev(struct dma_chan *chan)
 181 {
 182         return &chan->dev->device;
 183 }
 184
 185 static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d)
 186 {
 187         return container_of(d, struct sun6i_dma_dev, slave);
 188 }
 189
 190 static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan)
 191 {
 192         return container_of(chan, struct sun6i_vchan, vc.chan);
 193 }
 194
 195 static inline struct sun6i_desc *
 196 to_sun6i_desc(struct dma_async_tx_descriptor *tx)
 197 {
 198         return container_of(tx, struct sun6i_desc, vd.tx);
 199 }
 200
 201 static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev)
 202 {
 203         dev_dbg(sdev->slave.dev, "Common register:\n"
 204                 "\tmask0(%04x): 0x%08x\n"
 205                 "\tmask1(%04x): 0x%08x\n"
 206                 "\tpend0(%04x): 0x%08x\n"
 207                 "\tpend1(%04x): 0x%08x\n"
 208                 "\tstats(%04x): 0x%08x\n",
 209                 DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)),
 210                 DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)),
 211                 DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)),
 212                 DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)),
 213                 DMA_STAT, readl(sdev->base + DMA_STAT));
 214 }
 215
 216 static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev,
 217                                             struct sun6i_pchan *pchan)
 218 {
 219         phys_addr_t reg = virt_to_phys(pchan->base);
 220
 221         dev_dbg(sdev->slave.dev, "Chan %d reg: %pa\n"
 222                 "\t___en(%04x): \t0x%08x\n"
 223                 "\tpause(%04x): \t0x%08x\n"
 224                 "\tstart(%04x): \t0x%08x\n"
 225                 "\t__cfg(%04x): \t0x%08x\n"
 226                 "\t__src(%04x): \t0x%08x\n"
 227                 "\t__dst(%04x): \t0x%08x\n"
 228                 "\tcount(%04x): \t0x%08x\n"
 229                 "\t_para(%04x): \t0x%08x\n\n",
 230                 pchan->idx, &reg,
 231                 DMA_CHAN_ENABLE,
 232                 readl(pchan->base + DMA_CHAN_ENABLE),
 233                 DMA_CHAN_PAUSE,
 234                 readl(pchan->base + DMA_CHAN_PAUSE),
 235                 DMA_CHAN_LLI_ADDR,
 236                 readl(pchan->base + DMA_CHAN_LLI_ADDR),
 237                 DMA_CHAN_CUR_CFG,
 238                 readl(pchan->base + DMA_CHAN_CUR_CFG),
 239                 DMA_CHAN_CUR_SRC,
 240                 readl(pchan->base + DMA_CHAN_CUR_SRC),
 241                 DMA_CHAN_CUR_DST,
 242                 readl(pchan->base + DMA_CHAN_CUR_DST),
 243                 DMA_CHAN_CUR_CNT,
 244                 readl(pchan->base + DMA_CHAN_CUR_CNT),
 245                 DMA_CHAN_CUR_PARA,
 246                 readl(pchan->base + DMA_CHAN_CUR_PARA));
 247 }
 248
 249 static inline s8 convert_burst(u32 maxburst)
 250 {
 251         switch (maxburst) {
 252         case 1:
 253                 return 0;
 254         case 8:
 255                 return 2;
 256         default:
 257                 return -EINVAL;
 258         }
 259 }
 260
 261 static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width)
 262 {
 263         if ((addr_width < DMA_SLAVE_BUSWIDTH_1_BYTE) ||
 264             (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES))
 265                 return -EINVAL;
 266
 267         return addr_width >> 1;
 268 }
 269
 270 static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan)
 271 {
 272         struct sun6i_desc *txd = pchan->desc;
 273         struct sun6i_dma_lli *lli;
 274         size_t bytes;
 275         dma_addr_t pos;
 276
 277         pos = readl(pchan->base + DMA_CHAN_LLI_ADDR);
 278         bytes = readl(pchan->base + DMA_CHAN_CUR_CNT);
 279
 280         if (pos == LLI_LAST_ITEM)
 281                 return bytes;
 282
 283         for (lli = txd->v_lli; lli; lli = lli->v_lli_next) {
 284                 if (lli->p_lli_next == pos) {
 285                         for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next)
 286                                 bytes += lli->len;
 287                         break;
 288                 }
 289         }
 290
 291         return bytes;
 292 }
 293
 294 static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev,
 295                                struct sun6i_dma_lli *next,
 296                                dma_addr_t next_phy,
 297                                struct sun6i_desc *txd)
 298 {
 299         if ((!prev && !txd) || !next)
 300                 return NULL;
 301
 302         if (!prev) {
 303                 txd->p_lli = next_phy;
 304                 txd->v_lli = next;
 305         } else {
 306                 prev->p_lli_next = next_phy;
 307                 prev->v_lli_next = next;
 308         }
 309
 310         next->p_lli_next = LLI_LAST_ITEM;
 311         next->v_lli_next = NULL;
 312
 313         return next;
 314 }
 315
 316 static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
 317                                       struct sun6i_dma_lli *lli)
 318 {
 319         phys_addr_t p_lli = virt_to_phys(lli);
 320
 321         dev_dbg(chan2dev(&vchan->vc.chan),
 322                 "\n\tdesc:   p - %pa v - 0x%p\n"
 323                 "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n"
 324                 "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n",
 325                 &p_lli, lli,
 326                 lli->cfg, lli->src, lli->dst,
 327                 lli->len, lli->para, lli->p_lli_next);
 328 }
 329
 330 static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
 331 {
 332         struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
 333         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device);
 334         struct sun6i_dma_lli *v_lli, *v_next;
 335         dma_addr_t p_lli, p_next;
 336
 337         if (unlikely(!txd))
 338                 return;
 339
 340         p_lli = txd->p_lli;
 341         v_lli = txd->v_lli;
 342
 343         while (v_lli) {
 344                 v_next = v_lli->v_lli_next;
 345                 p_next = v_lli->p_lli_next;
 346
 347                 dma_pool_free(sdev->pool, v_lli, p_lli);
 348
 349                 v_lli = v_next;
 350                 p_lli = p_next;
 351         }
 352
 353         kfree(txd);
 354 }
 355
 356 static int sun6i_dma_start_desc(struct sun6i_vchan *vchan)
 357 {
 358         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device);
 359         struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc);
 360         struct sun6i_pchan *pchan = vchan->phy;
 361         u32 irq_val, irq_reg, irq_offset;
 362
 363         if (!pchan)
 364                 return -EAGAIN;
 365
 366         if (!desc) {
 367                 pchan->desc = NULL;
 368                 pchan->done = NULL;
 369                 return -EAGAIN;
 370         }
 371
 372         list_del(&desc->node);
 373
 374         pchan->desc = to_sun6i_desc(&desc->tx);
 375         pchan->done = NULL;
 376
 377         sun6i_dma_dump_lli(vchan, pchan->desc->v_lli);
 378
 379         irq_reg = pchan->idx / DMA_IRQ_CHAN_NR;
 380         irq_offset = pchan->idx % DMA_IRQ_CHAN_NR;
 381
 382         vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE;
 383
 384         irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg));
 385         irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) <<
 386                         (irq_offset * DMA_IRQ_CHAN_WIDTH));
 387         irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH);
 388         writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg));
 389
 390         writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR);
 391         writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE);
 392
 393         sun6i_dma_dump_com_regs(sdev);
 394         sun6i_dma_dump_chan_regs(sdev, pchan);
 395
 396         return 0;
 397 }
 398
 399 static void sun6i_dma_tasklet(unsigned long data)
 400 {
 401         struct sun6i_dma_dev *sdev = (struct sun6i_dma_dev *)data;
 402         const struct sun6i_dma_config *cfg = sdev->cfg;
 403         struct sun6i_vchan *vchan;
 404         struct sun6i_pchan *pchan;
 405         unsigned int pchan_alloc = 0;
 406         unsigned int pchan_idx;
 407
 408         list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) {
 409                 spin_lock_irq(&vchan->vc.lock);
 410
 411                 pchan = vchan->phy;
 412
 413                 if (pchan && pchan->done) {
 414                         if (sun6i_dma_start_desc(vchan)) {
 415                                 /*
 416                                  * No current txd associated with this channel
 417                                  */
 418                                 dev_dbg(sdev->slave.dev, "pchan %u: free\n",
 419                                         pchan->idx);
 420
 421                                 /* Mark this channel free */
 422                                 vchan->phy = NULL;
 423                                 pchan->vchan = NULL;
 424                         }
 425                 }
 426                 spin_unlock_irq(&vchan->vc.lock);
 427         }
 428
 429         spin_lock_irq(&sdev->lock);
 430         for (pchan_idx = 0; pchan_idx < cfg->nr_max_channels; pchan_idx++) {
 431                 pchan = &sdev->pchans[pchan_idx];
 432
 433                 if (pchan->vchan || list_empty(&sdev->pending))
 434                         continue;
 435
 436                 vchan = list_first_entry(&sdev->pending,
 437                                          struct sun6i_vchan, node);
 438
 439                 /* Remove from pending channels */
 440                 list_del_init(&vchan->node);
 441                 pchan_alloc |= BIT(pchan_idx);
 442
 443                 /* Mark this channel allocated */
 444                 pchan->vchan = vchan;
 445                 vchan->phy = pchan;
 446                 dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n",
 447                         pchan->idx, &vchan->vc);
 448         }
 449         spin_unlock_irq(&sdev->lock);
 450
 451         for (pchan_idx = 0; pchan_idx < cfg->nr_max_channels; pchan_idx++) {
 452                 if (!(pchan_alloc & BIT(pchan_idx)))
 453                         continue;
 454
 455                 pchan = sdev->pchans + pchan_idx;
 456                 vchan = pchan->vchan;
 457                 if (vchan) {
 458                         spin_lock_irq(&vchan->vc.lock);
 459                         sun6i_dma_start_desc(vchan);
 460                         spin_unlock_irq(&vchan->vc.lock);
 461                 }
 462         }
 463 }
 464
 465 static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id)
 466 {
 467         struct sun6i_dma_dev *sdev = dev_id;
 468         struct sun6i_vchan *vchan;
 469         struct sun6i_pchan *pchan;
 470         int i, j, ret = IRQ_NONE;
 471         u32 status;
 472
 473         for (i = 0; i < sdev->cfg->nr_max_channels / DMA_IRQ_CHAN_NR; i++) {
 474                 status = readl(sdev->base + DMA_IRQ_STAT(i));
 475                 if (!status)
 476                         continue;
 477
 478                 dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n",
 479                         i ? "high" : "low", status);
 480
 481                 writel(status, sdev->base + DMA_IRQ_STAT(i));
 482
 483                 for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) {
 484                         pchan = sdev->pchans + j;
 485                         vchan = pchan->vchan;
 486                         if (vchan && (status & vchan->irq_type)) {
 487                                 if (vchan->cyclic) {
 488                                         vchan_cyclic_callback(&pchan->desc->vd);
 489                                 } else {
 490                                         spin_lock(&vchan->vc.lock);
 491                                         vchan_cookie_complete(&pchan->desc->vd);
 492                                         pchan->done = pchan->desc;
 493                                         spin_unlock(&vchan->vc.lock);
 494                                 }
 495                         }
 496
 497                         status = status >> DMA_IRQ_CHAN_WIDTH;
 498                 }
 499
 500                 if (!atomic_read(&sdev->tasklet_shutdown))
 501                         tasklet_schedule(&sdev->task);
 502                 ret = IRQ_HANDLED;
 503         }
 504
 505         return ret;
 506 }
 507
 508 static int set_config(struct sun6i_dma_dev *sdev,
 509                         struct dma_slave_config *sconfig,
 510                         enum dma_transfer_direction direction,
 511                         u32 *p_cfg)
 512 {
 513         s8 src_width, dst_width, src_burst, dst_burst;
 514
 515         switch (direction) {
 516         case DMA_MEM_TO_DEV:
 517                 src_burst = convert_burst(sconfig->src_maxburst ?
 518                                         sconfig->src_maxburst : 8);
 519                 src_width = convert_buswidth(sconfig->src_addr_width !=
 520                                                 DMA_SLAVE_BUSWIDTH_UNDEFINED ?
 521                                 sconfig->src_addr_width :
 522                                 DMA_SLAVE_BUSWIDTH_4_BYTES);
 523                 dst_burst = convert_burst(sconfig->dst_maxburst);
 524                 dst_width = convert_buswidth(sconfig->dst_addr_width);
 525                 break;
 526         case DMA_DEV_TO_MEM:
 527                 src_burst = convert_burst(sconfig->src_maxburst);
 528                 src_width = convert_buswidth(sconfig->src_addr_width);
 529                 dst_burst = convert_burst(sconfig->dst_maxburst ?
 530                                         sconfig->dst_maxburst : 8);
 531                 dst_width = convert_buswidth(sconfig->dst_addr_width !=
 532                                                 DMA_SLAVE_BUSWIDTH_UNDEFINED ?
 533                                 sconfig->dst_addr_width :
 534                                 DMA_SLAVE_BUSWIDTH_4_BYTES);
 535                 break;
 536         default:
 537                 return -EINVAL;
 538         }
 539
 540         if (src_burst < 0)
 541                 return src_burst;
 542         if (src_width < 0)
 543                 return src_width;
 544         if (dst_burst < 0)
 545                 return dst_burst;
 546         if (dst_width < 0)
 547                 return dst_width;
 548
 549         *p_cfg = DMA_CHAN_CFG_SRC_BURST(src_burst) |
 550                 DMA_CHAN_CFG_SRC_WIDTH(src_width) |
 551                 DMA_CHAN_CFG_DST_BURST(dst_burst) |
 552                 DMA_CHAN_CFG_DST_WIDTH(dst_width);
 553
 554         return 0;
 555 }
 556
 557 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
 558                 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 559                 size_t len, unsigned long flags)
 560 {
 561         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 562         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 563         struct sun6i_dma_lli *v_lli;
 564         struct sun6i_desc *txd;
 565         dma_addr_t p_lli;
 566         s8 burst, width;
 567
 568         dev_dbg(chan2dev(chan),
 569                 "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
 570                 __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags);
 571
 572         if (!len)
 573                 return NULL;
 574
 575         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 576         if (!txd)
 577                 return NULL;
 578
 579         v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
 580         if (!v_lli) {
 581                 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
 582                 goto err_txd_free;
 583         }
 584
 585         v_lli->src = src;
 586         v_lli->dst = dest;
 587         v_lli->len = len;
 588         v_lli->para = NORMAL_WAIT;
 589
 590         burst = convert_burst(8);
 591         width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES);
 592         v_lli->cfg = DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
 593                 DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
 594                 DMA_CHAN_CFG_DST_LINEAR_MODE |
 595                 DMA_CHAN_CFG_SRC_LINEAR_MODE |
 596                 DMA_CHAN_CFG_SRC_BURST(burst) |
 597                 DMA_CHAN_CFG_SRC_WIDTH(width) |
 598                 DMA_CHAN_CFG_DST_BURST(burst) |
 599                 DMA_CHAN_CFG_DST_WIDTH(width);
 600
 601         sun6i_dma_lli_add(NULL, v_lli, p_lli, txd);
 602
 603         sun6i_dma_dump_lli(vchan, v_lli);
 604
 605         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 606
 607 err_txd_free:
 608         kfree(txd);
 609         return NULL;
 610 }
 611
 612 static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 613                 struct dma_chan *chan, struct scatterlist *sgl,
 614                 unsigned int sg_len, enum dma_transfer_direction dir,
 615                 unsigned long flags, void *context)
 616 {
 617         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 618         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 619         struct dma_slave_config *sconfig = &vchan->cfg;
 620         struct sun6i_dma_lli *v_lli, *prev = NULL;
 621         struct sun6i_desc *txd;
 622         struct scatterlist *sg;
 623         dma_addr_t p_lli;
 624         u32 lli_cfg;
 625         int i, ret;
 626
 627         if (!sgl)
 628                 return NULL;
 629
 630         ret = set_config(sdev, sconfig, dir, &lli_cfg);
 631         if (ret) {
 632                 dev_err(chan2dev(chan), "Invalid DMA configuration\n");
 633                 return NULL;
 634         }
 635
 636         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 637         if (!txd)
 638                 return NULL;
 639
 640         for_each_sg(sgl, sg, sg_len, i) {
 641                 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
 642                 if (!v_lli)
 643                         goto err_lli_free;
 644
 645                 v_lli->len = sg_dma_len(sg);
 646                 v_lli->para = NORMAL_WAIT;
 647
 648                 if (dir == DMA_MEM_TO_DEV) {
 649                         v_lli->src = sg_dma_address(sg);
 650                         v_lli->dst = sconfig->dst_addr;
 651                         v_lli->cfg = lli_cfg |
 652                                 DMA_CHAN_CFG_DST_IO_MODE |
 653                                 DMA_CHAN_CFG_SRC_LINEAR_MODE |
 654                                 DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
 655                                 DMA_CHAN_CFG_DST_DRQ(vchan->port);
 656
 657                         dev_dbg(chan2dev(chan),
 658                                 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 659                                 __func__, vchan->vc.chan.chan_id,
 660                                 &sconfig->dst_addr, &sg_dma_address(sg),
 661                                 sg_dma_len(sg), flags);
 662
 663                 } else {
 664                         v_lli->src = sconfig->src_addr;
 665                         v_lli->dst = sg_dma_address(sg);
 666                         v_lli->cfg = lli_cfg |
 667                                 DMA_CHAN_CFG_DST_LINEAR_MODE |
 668                                 DMA_CHAN_CFG_SRC_IO_MODE |
 669                                 DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
 670                                 DMA_CHAN_CFG_SRC_DRQ(vchan->port);
 671
 672                         dev_dbg(chan2dev(chan),
 673                                 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 674                                 __func__, vchan->vc.chan.chan_id,
 675                                 &sg_dma_address(sg), &sconfig->src_addr,
 676                                 sg_dma_len(sg), flags);
 677                 }
 678
 679                 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
 680         }
 681
 682         dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli);
 683         for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
 684                 sun6i_dma_dump_lli(vchan, prev);
 685
 686         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 687
 688 err_lli_free:
 689         for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
 690                 dma_pool_free(sdev->pool, prev, virt_to_phys(prev));
 691         kfree(txd);
 692         return NULL;
 693 }
 694
 695 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 696                                         struct dma_chan *chan,
 697                                         dma_addr_t buf_addr,
 698                                         size_t buf_len,
 699                                         size_t period_len,
 700                                         enum dma_transfer_direction dir,
 701                                         unsigned long flags)
 702 {
 703         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 704         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 705         struct dma_slave_config *sconfig = &vchan->cfg;
 706         struct sun6i_dma_lli *v_lli, *prev = NULL;
 707         struct sun6i_desc *txd;
 708         dma_addr_t p_lli;
 709         u32 lli_cfg;
 710         unsigned int i, periods = buf_len / period_len;
 711         int ret;
 712
 713         ret = set_config(sdev, sconfig, dir, &lli_cfg);
 714         if (ret) {
 715                 dev_err(chan2dev(chan), "Invalid DMA configuration\n");
 716                 return NULL;
 717         }
 718
 719         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 720         if (!txd)
 721                 return NULL;
 722
 723         for (i = 0; i < periods; i++) {
 724                 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
 725                 if (!v_lli) {
 726                         dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
 727                         goto err_lli_free;
 728                 }
 729
 730                 v_lli->len = period_len;
 731                 v_lli->para = NORMAL_WAIT;
 732
 733                 if (dir == DMA_MEM_TO_DEV) {
 734                         v_lli->src = buf_addr + period_len * i;
 735                         v_lli->dst = sconfig->dst_addr;
 736                         v_lli->cfg = lli_cfg |
 737                                 DMA_CHAN_CFG_DST_IO_MODE |
 738                                 DMA_CHAN_CFG_SRC_LINEAR_MODE |
 739                                 DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
 740                                 DMA_CHAN_CFG_DST_DRQ(vchan->port);
 741                 } else {
 742                         v_lli->src = sconfig->src_addr;
 743                         v_lli->dst = buf_addr + period_len * i;
 744                         v_lli->cfg = lli_cfg |
 745                                 DMA_CHAN_CFG_DST_LINEAR_MODE |
 746                                 DMA_CHAN_CFG_SRC_IO_MODE |
 747                                 DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
 748                                 DMA_CHAN_CFG_SRC_DRQ(vchan->port);
 749                 }
 750
 751                 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
 752         }
 753
 754         prev->p_lli_next = txd->p_lli;          /* cyclic list */
 755
 756         vchan->cyclic = true;
 757
 758         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 759
 760 err_lli_free:
 761         for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
 762                 dma_pool_free(sdev->pool, prev, virt_to_phys(prev));
 763         kfree(txd);
 764         return NULL;
 765 }
 766
 767 static int sun6i_dma_config(struct dma_chan *chan,
 768                             struct dma_slave_config *config)
 769 {
 770         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 771
 772         memcpy(&vchan->cfg, config, sizeof(*config));
 773
 774         return 0;
 775 }
 776
 777 static int sun6i_dma_pause(struct dma_chan *chan)
 778 {
 779         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 780         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 781         struct sun6i_pchan *pchan = vchan->phy;
 782
 783         dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc);
 784
 785         if (pchan) {
 786                 writel(DMA_CHAN_PAUSE_PAUSE,
 787                        pchan->base + DMA_CHAN_PAUSE);
 788         } else {
 789                 spin_lock(&sdev->lock);
 790                 list_del_init(&vchan->node);
 791                 spin_unlock(&sdev->lock);
 792         }
 793
 794         return 0;
 795 }
 796
 797 static int sun6i_dma_resume(struct dma_chan *chan)
 798 {
 799         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 800         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 801         struct sun6i_pchan *pchan = vchan->phy;
 802         unsigned long flags;
 803
 804         dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
 805
 806         spin_lock_irqsave(&vchan->vc.lock, flags);
 807
 808         if (pchan) {
 809                 writel(DMA_CHAN_PAUSE_RESUME,
 810                        pchan->base + DMA_CHAN_PAUSE);
 811         } else if (!list_empty(&vchan->vc.desc_issued)) {
 812                 spin_lock(&sdev->lock);
 813                 list_add_tail(&vchan->node, &sdev->pending);
 814                 spin_unlock(&sdev->lock);
 815         }
 816
 817         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 818
 819         return 0;
 820 }
 821
 822 static int sun6i_dma_terminate_all(struct dma_chan *chan)
 823 {
 824         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 825         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 826         struct sun6i_pchan *pchan = vchan->phy;
 827         unsigned long flags;
 828         LIST_HEAD(head);
 829
 830         spin_lock(&sdev->lock);
 831         list_del_init(&vchan->node);
 832         spin_unlock(&sdev->lock);
 833
 834         spin_lock_irqsave(&vchan->vc.lock, flags);
 835
 836         if (vchan->cyclic) {
 837                 vchan->cyclic = false;
 838                 if (pchan && pchan->desc) {
 839                         struct virt_dma_desc *vd = &pchan->desc->vd;
 840                         struct virt_dma_chan *vc = &vchan->vc;
 841
 842                         list_add_tail(&vd->node, &vc->desc_completed);
 843                 }
 844         }
 845
 846         vchan_get_all_descriptors(&vchan->vc, &head);
 847
 848         if (pchan) {
 849                 writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE);
 850                 writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE);
 851
 852                 vchan->phy = NULL;
 853                 pchan->vchan = NULL;
 854                 pchan->desc = NULL;
 855                 pchan->done = NULL;
 856         }
 857
 858         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 859
 860         vchan_dma_desc_free_list(&vchan->vc, &head);
 861
 862         return 0;
 863 }
 864
 865 static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan,
 866                                            dma_cookie_t cookie,
 867                                            struct dma_tx_state *state)
 868 {
 869         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 870         struct sun6i_pchan *pchan = vchan->phy;
 871         struct sun6i_dma_lli *lli;
 872         struct virt_dma_desc *vd;
 873         struct sun6i_desc *txd;
 874         enum dma_status ret;
 875         unsigned long flags;
 876         size_t bytes = 0;
 877
 878         ret = dma_cookie_status(chan, cookie, state);
 879         if (ret == DMA_COMPLETE || !state)
 880                 return ret;
 881
 882         spin_lock_irqsave(&vchan->vc.lock, flags);
 883
 884         vd = vchan_find_desc(&vchan->vc, cookie);
 885         txd = to_sun6i_desc(&vd->tx);
 886
 887         if (vd) {
 888                 for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next)
 889                         bytes += lli->len;
 890         } else if (!pchan || !pchan->desc) {
 891                 bytes = 0;
 892         } else {
 893                 bytes = sun6i_get_chan_size(pchan);
 894         }
 895
 896         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 897
 898         dma_set_residue(state, bytes);
 899
 900         return ret;
 901 }
 902
 903 static void sun6i_dma_issue_pending(struct dma_chan *chan)
 904 {
 905         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 906         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 907         unsigned long flags;
 908
 909         spin_lock_irqsave(&vchan->vc.lock, flags);
 910
 911         if (vchan_issue_pending(&vchan->vc)) {
 912                 spin_lock(&sdev->lock);
 913
 914                 if (!vchan->phy && list_empty(&vchan->node)) {
 915                         list_add_tail(&vchan->node, &sdev->pending);
 916                         tasklet_schedule(&sdev->task);
 917                         dev_dbg(chan2dev(chan), "vchan %p: issued\n",
 918                                 &vchan->vc);
 919                 }
 920
 921                 spin_unlock(&sdev->lock);
 922         } else {
 923                 dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n",
 924                         &vchan->vc);
 925         }
 926
 927         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 928 }
 929
 930 static void sun6i_dma_free_chan_resources(struct dma_chan *chan)
 931 {
 932         struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 933         struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 934         unsigned long flags;
 935
 936         spin_lock_irqsave(&sdev->lock, flags);
 937         list_del_init(&vchan->node);
 938         spin_unlock_irqrestore(&sdev->lock, flags);
 939
 940         vchan_free_chan_resources(&vchan->vc);
 941 }
 942
 943 static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec,
 944                                            struct of_dma *ofdma)
 945 {
 946         struct sun6i_dma_dev *sdev = ofdma->of_dma_data;
 947         struct sun6i_vchan *vchan;
 948         struct dma_chan *chan;
 949         u8 port = dma_spec->args[0];
 950
 951         if (port > sdev->cfg->nr_max_requests)
 952                 return NULL;
 953
 954         chan = dma_get_any_slave_channel(&sdev->slave);
 955         if (!chan)
 956                 return NULL;
 957
 958         vchan = to_sun6i_vchan(chan);
 959         vchan->port = port;
 960
 961         return chan;
 962 }
 963
 964 static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev)
 965 {
 966         /* Disable all interrupts from DMA */
 967         writel(0, sdev->base + DMA_IRQ_EN(0));
 968         writel(0, sdev->base + DMA_IRQ_EN(1));
 969
 970         /* Prevent spurious interrupts from scheduling the tasklet */
 971         atomic_inc(&sdev->tasklet_shutdown);
 972
 973         /* Make sure we won't have any further interrupts */
 974         devm_free_irq(sdev->slave.dev, sdev->irq, sdev);
 975
 976         /* Actually prevent the tasklet from being scheduled */
 977         tasklet_kill(&sdev->task);
 978 }
 979
 980 static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev)
 981 {
 982         int i;
 983
 984         for (i = 0; i < sdev->cfg->nr_max_vchans; i++) {
 985                 struct sun6i_vchan *vchan = &sdev->vchans[i];
 986
 987                 list_del(&vchan->vc.chan.device_node);
 988                 tasklet_kill(&vchan->vc.task);
 989         }
 990 }
 991
 992 /*
 993  * For A31:
 994  *
 995  * There's 16 physical channels that can work in parallel.
 996  *
 997  * However we have 30 different endpoints for our requests.
 998  *
 999  * Since the channels are able to handle only an unidirectional
1000  * transfer, we need to allocate more virtual channels so that
1001  * everyone can grab one channel.
1002  *
1003  * Some devices can't work in both direction (mostly because it
1004  * wouldn't make sense), so we have a bit fewer virtual channels than
1005  * 2 channels per endpoints.
1006  */
1007
1008 static struct sun6i_dma_config sun6i_a31_dma_cfg = {
1009         .nr_max_channels = 16,
1010         .nr_max_requests = 30,
1011         .nr_max_vchans   = 53,
1012 };
1013
1014 /*
1015  * The A23 only has 8 physical channels, a maximum DRQ port id of 24,
1016  * and a total of 37 usable source and destination endpoints.
1017  */
1018
1019 static struct sun6i_dma_config sun8i_a23_dma_cfg = {
1020         .nr_max_channels = 8,
1021         .nr_max_requests = 24,
1022         .nr_max_vchans   = 37,
1023         .gate_needed     = true,
1024 };
1025
1026 static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
1027         .nr_max_channels = 8,
1028         .nr_max_requests = 28,
1029         .nr_max_vchans   = 39,
1030 };
1031
1032 /*
1033  * The H3 has 12 physical channels, a maximum DRQ port id of 27,
1034  * and a total of 34 usable source and destination endpoints.
1035  */
1036
1037 static struct sun6i_dma_config sun8i_h3_dma_cfg = {
1038         .nr_max_channels = 12,
1039         .nr_max_requests = 27,
1040         .nr_max_vchans   = 34,
1041 };
1042
1043 /*
1044  * The V3s have only 8 physical channels, a maximum DRQ port id of 23,
1045  * and a total of 24 usable source and destination endpoints.
1046  */
1047
1048 static struct sun6i_dma_config sun8i_v3s_dma_cfg = {
1049         .nr_max_channels = 8,
1050         .nr_max_requests = 23,
1051         .nr_max_vchans   = 24,
1052         .gate_needed     = true,
1053 };
1054
1055 static const struct of_device_id sun6i_dma_match[] = {
1056         { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg },
1057         { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg },
1058         { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
1059         { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
1060         { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
1061         { /* sentinel */ }
1062 };
1063 MODULE_DEVICE_TABLE(of, sun6i_dma_match);
1064
1065 static int sun6i_dma_probe(struct platform_device *pdev)
1066 {
1067         const struct of_device_id *device;
1068         struct sun6i_dma_dev *sdc;
1069         struct resource *res;
1070         int ret, i;
1071
1072         sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL);
1073         if (!sdc)
1074                 return -ENOMEM;
1075
1076         device = of_match_device(sun6i_dma_match, &pdev->dev);
1077         if (!device)
1078                 return -ENODEV;
1079         sdc->cfg = device->data;
1080
1081         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1082         sdc->base = devm_ioremap_resource(&pdev->dev, res);
1083         if (IS_ERR(sdc->base))
1084                 return PTR_ERR(sdc->base);
1085
1086         sdc->irq = platform_get_irq(pdev, 0);
1087         if (sdc->irq < 0) {
1088                 dev_err(&pdev->dev, "Cannot claim IRQ\n");
1089                 return sdc->irq;
1090         }
1091
1092         sdc->clk = devm_clk_get(&pdev->dev, NULL);
1093         if (IS_ERR(sdc->clk)) {
1094                 dev_err(&pdev->dev, "No clock specified\n");
1095                 return PTR_ERR(sdc->clk);
1096         }
1097
1098         sdc->rstc = devm_reset_control_get(&pdev->dev, NULL);
1099         if (IS_ERR(sdc->rstc)) {
1100                 dev_err(&pdev->dev, "No reset controller specified\n");
1101                 return PTR_ERR(sdc->rstc);
1102         }
1103
1104         sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
1105                                      sizeof(struct sun6i_dma_lli), 4, 0);
1106         if (!sdc->pool) {
1107                 dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
1108                 return -ENOMEM;
1109         }
1110
1111         platform_set_drvdata(pdev, sdc);
1112         INIT_LIST_HEAD(&sdc->pending);
1113         spin_lock_init(&sdc->lock);
1114
1115         dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask);
1116         dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask);
1117         dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask);
1118         dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask);
1119
1120         INIT_LIST_HEAD(&sdc->slave.channels);
1121         sdc->slave.device_free_chan_resources   = sun6i_dma_free_chan_resources;
1122         sdc->slave.device_tx_status             = sun6i_dma_tx_status;
1123         sdc->slave.device_issue_pending         = sun6i_dma_issue_pending;
1124         sdc->slave.device_prep_slave_sg         = sun6i_dma_prep_slave_sg;
1125         sdc->slave.device_prep_dma_memcpy       = sun6i_dma_prep_dma_memcpy;
1126         sdc->slave.device_prep_dma_cyclic       = sun6i_dma_prep_dma_cyclic;
1127         sdc->slave.copy_align                   = DMAENGINE_ALIGN_4_BYTES;
1128         sdc->slave.device_config                = sun6i_dma_config;
1129         sdc->slave.device_pause                 = sun6i_dma_pause;
1130         sdc->slave.device_resume                = sun6i_dma_resume;
1131         sdc->slave.device_terminate_all         = sun6i_dma_terminate_all;
1132         sdc->slave.src_addr_widths              = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1133                                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1134                                                   BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1135         sdc->slave.dst_addr_widths              = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1136                                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1137                                                   BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1138         sdc->slave.directions                   = BIT(DMA_DEV_TO_MEM) |
1139                                                   BIT(DMA_MEM_TO_DEV);
1140         sdc->slave.residue_granularity          = DMA_RESIDUE_GRANULARITY_BURST;
1141         sdc->slave.dev = &pdev->dev;
1142
1143         sdc->pchans = devm_kcalloc(&pdev->dev, sdc->cfg->nr_max_channels,
1144                                    sizeof(struct sun6i_pchan), GFP_KERNEL);
1145         if (!sdc->pchans)
1146                 return -ENOMEM;
1147
1148         sdc->vchans = devm_kcalloc(&pdev->dev, sdc->cfg->nr_max_vchans,
1149                                    sizeof(struct sun6i_vchan), GFP_KERNEL);
1150         if (!sdc->vchans)
1151                 return -ENOMEM;
1152
1153         tasklet_init(&sdc->task, sun6i_dma_tasklet, (unsigned long)sdc);
1154
1155         for (i = 0; i < sdc->cfg->nr_max_channels; i++) {
1156                 struct sun6i_pchan *pchan = &sdc->pchans[i];
1157
1158                 pchan->idx = i;
1159                 pchan->base = sdc->base + 0x100 + i * 0x40;
1160         }
1161
1162         for (i = 0; i < sdc->cfg->nr_max_vchans; i++) {
1163                 struct sun6i_vchan *vchan = &sdc->vchans[i];
1164
1165                 INIT_LIST_HEAD(&vchan->node);
1166                 vchan->vc.desc_free = sun6i_dma_free_desc;
1167                 vchan_init(&vchan->vc, &sdc->slave);
1168         }
1169
1170         ret = reset_control_deassert(sdc->rstc);
1171         if (ret) {
1172                 dev_err(&pdev->dev, "Couldn't deassert the device from reset\n");
1173                 goto err_chan_free;
1174         }
1175
1176         ret = clk_prepare_enable(sdc->clk);
1177         if (ret) {
1178                 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1179                 goto err_reset_assert;
1180         }
1181
1182         ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0,
1183                                dev_name(&pdev->dev), sdc);
1184         if (ret) {
1185                 dev_err(&pdev->dev, "Cannot request IRQ\n");
1186                 goto err_clk_disable;
1187         }
1188
1189         ret = dma_async_device_register(&sdc->slave);
1190         if (ret) {
1191                 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1192                 goto err_irq_disable;
1193         }
1194
1195         ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate,
1196                                          sdc);
1197         if (ret) {
1198                 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1199                 goto err_dma_unregister;
1200         }
1201
1202         if (sdc->cfg->gate_needed)
1203                 writel(SUN8I_DMA_GATE_ENABLE, sdc->base + SUN8I_DMA_GATE);
1204
1205         return 0;
1206
1207 err_dma_unregister:
1208         dma_async_device_unregister(&sdc->slave);
1209 err_irq_disable:
1210         sun6i_kill_tasklet(sdc);
1211 err_clk_disable:
1212         clk_disable_unprepare(sdc->clk);
1213 err_reset_assert:
1214         reset_control_assert(sdc->rstc);
1215 err_chan_free:
1216         sun6i_dma_free(sdc);
1217         return ret;
1218 }
1219
1220 static int sun6i_dma_remove(struct platform_device *pdev)
1221 {
1222         struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev);
1223
1224         of_dma_controller_free(pdev->dev.of_node);
1225         dma_async_device_unregister(&sdc->slave);
1226
1227         sun6i_kill_tasklet(sdc);
1228
1229         clk_disable_unprepare(sdc->clk);
1230         reset_control_assert(sdc->rstc);
1231
1232         sun6i_dma_free(sdc);
1233
1234         return 0;
1235 }
1236
1237 static struct platform_driver sun6i_dma_driver = {
1238         .probe          = sun6i_dma_probe,
1239         .remove         = sun6i_dma_remove,
1240         .driver = {
1241                 .name           = "sun6i-dma",
1242                 .of_match_table = sun6i_dma_match,
1243         },
1244 };
1245 module_platform_driver(sun6i_dma_driver);
1246
1247 MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver");
1248 MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>");
1249 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
1250 MODULE_LICENSE("GPL");