drivers/dma/sa11x0-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * SA11x0 DMAengine support
   4  *
   5  * Copyright (C) 2012 Russell King
   6  *   Derived in part from arch/arm/mach-sa1100/dma.c,
   7  *   Copyright (C) 2000, 2001 by Nicolas Pitre
   8  */
   9 #include <linux/sched.h>
  10 #include <linux/device.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/init.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/slab.h>
  18 #include <linux/spinlock.h>
  19
  20 #include "virt-dma.h"
  21
  22 #define NR_PHY_CHAN     6
  23 #define DMA_ALIGN       3
  24 #define DMA_MAX_SIZE    0x1fff
  25 #define DMA_CHUNK_SIZE  0x1000
  26
  27 #define DMA_DDAR        0x00
  28 #define DMA_DCSR_S      0x04
  29 #define DMA_DCSR_C      0x08
  30 #define DMA_DCSR_R      0x0c
  31 #define DMA_DBSA        0x10
  32 #define DMA_DBTA        0x14
  33 #define DMA_DBSB        0x18
  34 #define DMA_DBTB        0x1c
  35 #define DMA_SIZE        0x20
  36
  37 #define DCSR_RUN        (1 << 0)
  38 #define DCSR_IE         (1 << 1)
  39 #define DCSR_ERROR      (1 << 2)
  40 #define DCSR_DONEA      (1 << 3)
  41 #define DCSR_STRTA      (1 << 4)
  42 #define DCSR_DONEB      (1 << 5)
  43 #define DCSR_STRTB      (1 << 6)
  44 #define DCSR_BIU        (1 << 7)
  45
  46 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  47 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  48 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  49 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  50 #define DDAR_Ser0UDCTr  (0x0 << 4)
  51 #define DDAR_Ser0UDCRc  (0x1 << 4)
  52 #define DDAR_Ser1SDLCTr (0x2 << 4)
  53 #define DDAR_Ser1SDLCRc (0x3 << 4)
  54 #define DDAR_Ser1UARTTr (0x4 << 4)
  55 #define DDAR_Ser1UARTRc (0x5 << 4)
  56 #define DDAR_Ser2ICPTr  (0x6 << 4)
  57 #define DDAR_Ser2ICPRc  (0x7 << 4)
  58 #define DDAR_Ser3UARTTr (0x8 << 4)
  59 #define DDAR_Ser3UARTRc (0x9 << 4)
  60 #define DDAR_Ser4MCP0Tr (0xa << 4)
  61 #define DDAR_Ser4MCP0Rc (0xb << 4)
  62 #define DDAR_Ser4MCP1Tr (0xc << 4)
  63 #define DDAR_Ser4MCP1Rc (0xd << 4)
  64 #define DDAR_Ser4SSPTr  (0xe << 4)
  65 #define DDAR_Ser4SSPRc  (0xf << 4)
  66
  67 struct sa11x0_dma_sg {
  68         u32                     addr;
  69         u32                     len;
  70 };
  71
  72 struct sa11x0_dma_desc {
  73         struct virt_dma_desc    vd;
  74
  75         u32                     ddar;
  76         size_t                  size;
  77         unsigned                period;
  78         bool                    cyclic;
  79
  80         unsigned                sglen;
  81         struct sa11x0_dma_sg    sg[];
  82 };
  83
  84 struct sa11x0_dma_phy;
  85
  86 struct sa11x0_dma_chan {
  87         struct virt_dma_chan    vc;
  88
  89         /* protected by c->vc.lock */
  90         struct sa11x0_dma_phy   *phy;
  91         enum dma_status         status;
  92
  93         /* protected by d->lock */
  94         struct list_head        node;
  95
  96         u32                     ddar;
  97         const char              *name;
  98 };
  99
 100 struct sa11x0_dma_phy {
 101         void __iomem            *base;
 102         struct sa11x0_dma_dev   *dev;
 103         unsigned                num;
 104
 105         struct sa11x0_dma_chan  *vchan;
 106
 107         /* Protected by c->vc.lock */
 108         unsigned                sg_load;
 109         struct sa11x0_dma_desc  *txd_load;
 110         unsigned                sg_done;
 111         struct sa11x0_dma_desc  *txd_done;
 112         u32                     dbs[2];
 113         u32                     dbt[2];
 114         u32                     dcsr;
 115 };
 116
 117 struct sa11x0_dma_dev {
 118         struct dma_device       slave;
 119         void __iomem            *base;
 120         spinlock_t              lock;
 121         struct tasklet_struct   task;
 122         struct list_head        chan_pending;
 123         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 124 };
 125
 126 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 127 {
 128         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 129 }
 130
 131 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 132 {
 133         return container_of(dmadev, struct sa11x0_dma_dev, slave);
 134 }
 135
 136 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 137 {
 138         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 139
 140         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 141 }
 142
 143 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 144 {
 145         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 146 }
 147
 148 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 149 {
 150         list_del(&txd->vd.node);
 151         p->txd_load = txd;
 152         p->sg_load = 0;
 153
 154         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 155                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 156 }
 157
 158 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 159         struct sa11x0_dma_chan *c)
 160 {
 161         struct sa11x0_dma_desc *txd = p->txd_load;
 162         struct sa11x0_dma_sg *sg;
 163         void __iomem *base = p->base;
 164         unsigned dbsx, dbtx;
 165         u32 dcsr;
 166
 167         if (!txd)
 168                 return;
 169
 170         dcsr = readl_relaxed(base + DMA_DCSR_R);
 171
 172         /* Don't try to load the next transfer if both buffers are started */
 173         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 174                 return;
 175
 176         if (p->sg_load == txd->sglen) {
 177                 if (!txd->cyclic) {
 178                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 179
 180                         /*
 181                          * We have reached the end of the current descriptor.
 182                          * Peek at the next descriptor, and if compatible with
 183                          * the current, start processing it.
 184                          */
 185                         if (txn && txn->ddar == txd->ddar) {
 186                                 txd = txn;
 187                                 sa11x0_dma_start_desc(p, txn);
 188                         } else {
 189                                 p->txd_load = NULL;
 190                                 return;
 191                         }
 192                 } else {
 193                         /* Cyclic: reset back to beginning */
 194                         p->sg_load = 0;
 195                 }
 196         }
 197
 198         sg = &txd->sg[p->sg_load++];
 199
 200         /* Select buffer to load according to channel status */
 201         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 202             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 203                 dbsx = DMA_DBSA;
 204                 dbtx = DMA_DBTA;
 205                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 206         } else {
 207                 dbsx = DMA_DBSB;
 208                 dbtx = DMA_DBTB;
 209                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 210         }
 211
 212         writel_relaxed(sg->addr, base + dbsx);
 213         writel_relaxed(sg->len, base + dbtx);
 214         writel(dcsr, base + DMA_DCSR_S);
 215
 216         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 217                 p->num, dcsr,
 218                 'A' + (dbsx == DMA_DBSB), sg->addr,
 219                 'A' + (dbtx == DMA_DBTB), sg->len);
 220 }
 221
 222 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 223         struct sa11x0_dma_chan *c)
 224 {
 225         struct sa11x0_dma_desc *txd = p->txd_done;
 226
 227         if (++p->sg_done == txd->sglen) {
 228                 if (!txd->cyclic) {
 229                         vchan_cookie_complete(&txd->vd);
 230
 231                         p->sg_done = 0;
 232                         p->txd_done = p->txd_load;
 233
 234                         if (!p->txd_done)
 235                                 tasklet_schedule(&p->dev->task);
 236                 } else {
 237                         if ((p->sg_done % txd->period) == 0)
 238                                 vchan_cyclic_callback(&txd->vd);
 239
 240                         /* Cyclic: reset back to beginning */
 241                         p->sg_done = 0;
 242                 }
 243         }
 244
 245         sa11x0_dma_start_sg(p, c);
 246 }
 247
 248 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 249 {
 250         struct sa11x0_dma_phy *p = dev_id;
 251         struct sa11x0_dma_dev *d = p->dev;
 252         struct sa11x0_dma_chan *c;
 253         u32 dcsr;
 254
 255         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 256         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 257                 return IRQ_NONE;
 258
 259         /* Clear reported status bits */
 260         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 261                 p->base + DMA_DCSR_C);
 262
 263         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 264
 265         if (dcsr & DCSR_ERROR) {
 266                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 267                         p->num, dcsr,
 268                         readl_relaxed(p->base + DMA_DDAR),
 269                         readl_relaxed(p->base + DMA_DBSA),
 270                         readl_relaxed(p->base + DMA_DBTA),
 271                         readl_relaxed(p->base + DMA_DBSB),
 272                         readl_relaxed(p->base + DMA_DBTB));
 273         }
 274
 275         c = p->vchan;
 276         if (c) {
 277                 unsigned long flags;
 278
 279                 spin_lock_irqsave(&c->vc.lock, flags);
 280                 /*
 281                  * Now that we're holding the lock, check that the vchan
 282                  * really is associated with this pchan before touching the
 283                  * hardware.  This should always succeed, because we won't
 284                  * change p->vchan or c->phy while the channel is actively
 285                  * transferring.
 286                  */
 287                 if (c->phy == p) {
 288                         if (dcsr & DCSR_DONEA)
 289                                 sa11x0_dma_complete(p, c);
 290                         if (dcsr & DCSR_DONEB)
 291                                 sa11x0_dma_complete(p, c);
 292                 }
 293                 spin_unlock_irqrestore(&c->vc.lock, flags);
 294         }
 295
 296         return IRQ_HANDLED;
 297 }
 298
 299 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 300 {
 301         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 302
 303         /* If the issued list is empty, we have no further txds to process */
 304         if (txd) {
 305                 struct sa11x0_dma_phy *p = c->phy;
 306
 307                 sa11x0_dma_start_desc(p, txd);
 308                 p->txd_done = txd;
 309                 p->sg_done = 0;
 310
 311                 /* The channel should not have any transfers started */
 312                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 313                                       (DCSR_STRTA | DCSR_STRTB));
 314
 315                 /* Clear the run and start bits before changing DDAR */
 316                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 317                                p->base + DMA_DCSR_C);
 318                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 319
 320                 /* Try to start both buffers */
 321                 sa11x0_dma_start_sg(p, c);
 322                 sa11x0_dma_start_sg(p, c);
 323         }
 324 }
 325
 326 static void sa11x0_dma_tasklet(struct tasklet_struct *t)
 327 {
 328         struct sa11x0_dma_dev *d = from_tasklet(d, t, task);
 329         struct sa11x0_dma_phy *p;
 330         struct sa11x0_dma_chan *c;
 331         unsigned pch, pch_alloc = 0;
 332
 333         dev_dbg(d->slave.dev, "tasklet enter\n");
 334
 335         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 336                 spin_lock_irq(&c->vc.lock);
 337                 p = c->phy;
 338                 if (p && !p->txd_done) {
 339                         sa11x0_dma_start_txd(c);
 340                         if (!p->txd_done) {
 341                                 /* No current txd associated with this channel */
 342                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 343
 344                                 /* Mark this channel free */
 345                                 c->phy = NULL;
 346                                 p->vchan = NULL;
 347                         }
 348                 }
 349                 spin_unlock_irq(&c->vc.lock);
 350         }
 351
 352         spin_lock_irq(&d->lock);
 353         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 354                 p = &d->phy[pch];
 355
 356                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 357                         c = list_first_entry(&d->chan_pending,
 358                                 struct sa11x0_dma_chan, node);
 359                         list_del_init(&c->node);
 360
 361                         pch_alloc |= 1 << pch;
 362
 363                         /* Mark this channel allocated */
 364                         p->vchan = c;
 365
 366                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 367                 }
 368         }
 369         spin_unlock_irq(&d->lock);
 370
 371         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 372                 if (pch_alloc & (1 << pch)) {
 373                         p = &d->phy[pch];
 374                         c = p->vchan;
 375
 376                         spin_lock_irq(&c->vc.lock);
 377                         c->phy = p;
 378
 379                         sa11x0_dma_start_txd(c);
 380                         spin_unlock_irq(&c->vc.lock);
 381                 }
 382         }
 383
 384         dev_dbg(d->slave.dev, "tasklet exit\n");
 385 }
 386
 387
 388 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 389 {
 390         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 391         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 392         unsigned long flags;
 393
 394         spin_lock_irqsave(&d->lock, flags);
 395         list_del_init(&c->node);
 396         spin_unlock_irqrestore(&d->lock, flags);
 397
 398         vchan_free_chan_resources(&c->vc);
 399 }
 400
 401 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 402 {
 403         unsigned reg;
 404         u32 dcsr;
 405
 406         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 407
 408         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 409             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 410                 reg = DMA_DBSA;
 411         else
 412                 reg = DMA_DBSB;
 413
 414         return readl_relaxed(p->base + reg);
 415 }
 416
 417 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 418         dma_cookie_t cookie, struct dma_tx_state *state)
 419 {
 420         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 421         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 422         struct sa11x0_dma_phy *p;
 423         struct virt_dma_desc *vd;
 424         unsigned long flags;
 425         enum dma_status ret;
 426
 427         ret = dma_cookie_status(&c->vc.chan, cookie, state);
 428         if (ret == DMA_COMPLETE)
 429                 return ret;
 430
 431         if (!state)
 432                 return c->status;
 433
 434         spin_lock_irqsave(&c->vc.lock, flags);
 435         p = c->phy;
 436
 437         /*
 438          * If the cookie is on our issue queue, then the residue is
 439          * its total size.
 440          */
 441         vd = vchan_find_desc(&c->vc, cookie);
 442         if (vd) {
 443                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 444         } else if (!p) {
 445                 state->residue = 0;
 446         } else {
 447                 struct sa11x0_dma_desc *txd;
 448                 size_t bytes = 0;
 449
 450                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 451                         txd = p->txd_done;
 452                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 453                         txd = p->txd_load;
 454                 else
 455                         txd = NULL;
 456
 457                 ret = c->status;
 458                 if (txd) {
 459                         dma_addr_t addr = sa11x0_dma_pos(p);
 460                         unsigned i;
 461
 462                         dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
 463
 464                         for (i = 0; i < txd->sglen; i++) {
 465                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 466                                         i, txd->sg[i].addr, txd->sg[i].len);
 467                                 if (addr >= txd->sg[i].addr &&
 468                                     addr < txd->sg[i].addr + txd->sg[i].len) {
 469                                         unsigned len;
 470
 471                                         len = txd->sg[i].len -
 472                                                 (addr - txd->sg[i].addr);
 473                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 474                                                 i, len);
 475                                         bytes += len;
 476                                         i++;
 477                                         break;
 478                                 }
 479                         }
 480                         for (; i < txd->sglen; i++) {
 481                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 482                                         i, txd->sg[i].addr, txd->sg[i].len);
 483                                 bytes += txd->sg[i].len;
 484                         }
 485                 }
 486                 state->residue = bytes;
 487         }
 488         spin_unlock_irqrestore(&c->vc.lock, flags);
 489
 490         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
 491
 492         return ret;
 493 }
 494
 495 /*
 496  * Move pending txds to the issued list, and re-init pending list.
 497  * If not already pending, add this channel to the list of pending
 498  * channels and trigger the tasklet to run.
 499  */
 500 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 501 {
 502         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 503         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 504         unsigned long flags;
 505
 506         spin_lock_irqsave(&c->vc.lock, flags);
 507         if (vchan_issue_pending(&c->vc)) {
 508                 if (!c->phy) {
 509                         spin_lock(&d->lock);
 510                         if (list_empty(&c->node)) {
 511                                 list_add_tail(&c->node, &d->chan_pending);
 512                                 tasklet_schedule(&d->task);
 513                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 514                         }
 515                         spin_unlock(&d->lock);
 516                 }
 517         } else
 518                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 519         spin_unlock_irqrestore(&c->vc.lock, flags);
 520 }
 521
 522 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 523         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 524         enum dma_transfer_direction dir, unsigned long flags, void *context)
 525 {
 526         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 527         struct sa11x0_dma_desc *txd;
 528         struct scatterlist *sgent;
 529         unsigned i, j = sglen;
 530         size_t size = 0;
 531
 532         /* SA11x0 channels can only operate in their native direction */
 533         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 534                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 535                         &c->vc, c->ddar, dir);
 536                 return NULL;
 537         }
 538
 539         /* Do not allow zero-sized txds */
 540         if (sglen == 0)
 541                 return NULL;
 542
 543         for_each_sg(sg, sgent, sglen, i) {
 544                 dma_addr_t addr = sg_dma_address(sgent);
 545                 unsigned int len = sg_dma_len(sgent);
 546
 547                 if (len > DMA_MAX_SIZE)
 548                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 549                 if (addr & DMA_ALIGN) {
 550                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
 551                                 &c->vc, &addr);
 552                         return NULL;
 553                 }
 554         }
 555
 556         txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
 557         if (!txd) {
 558                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 559                 return NULL;
 560         }
 561
 562         j = 0;
 563         for_each_sg(sg, sgent, sglen, i) {
 564                 dma_addr_t addr = sg_dma_address(sgent);
 565                 unsigned len = sg_dma_len(sgent);
 566
 567                 size += len;
 568
 569                 do {
 570                         unsigned tlen = len;
 571
 572                         /*
 573                          * Check whether the transfer will fit.  If not, try
 574                          * to split the transfer up such that we end up with
 575                          * equal chunks - but make sure that we preserve the
 576                          * alignment.  This avoids small segments.
 577                          */
 578                         if (tlen > DMA_MAX_SIZE) {
 579                                 unsigned mult = DIV_ROUND_UP(tlen,
 580                                         DMA_MAX_SIZE & ~DMA_ALIGN);
 581
 582                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 583                         }
 584
 585                         txd->sg[j].addr = addr;
 586                         txd->sg[j].len = tlen;
 587
 588                         addr += tlen;
 589                         len -= tlen;
 590                         j++;
 591                 } while (len);
 592         }
 593
 594         txd->ddar = c->ddar;
 595         txd->size = size;
 596         txd->sglen = j;
 597
 598         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
 599                 &c->vc, &txd->vd, txd->size, txd->sglen);
 600
 601         return vchan_tx_prep(&c->vc, &txd->vd, flags);
 602 }
 603
 604 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 605         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 606         enum dma_transfer_direction dir, unsigned long flags)
 607 {
 608         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 609         struct sa11x0_dma_desc *txd;
 610         unsigned i, j, k, sglen, sgperiod;
 611
 612         /* SA11x0 channels can only operate in their native direction */
 613         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 614                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 615                         &c->vc, c->ddar, dir);
 616                 return NULL;
 617         }
 618
 619         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 620         sglen = size * sgperiod / period;
 621
 622         /* Do not allow zero-sized txds */
 623         if (sglen == 0)
 624                 return NULL;
 625
 626         txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
 627         if (!txd) {
 628                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 629                 return NULL;
 630         }
 631
 632         for (i = k = 0; i < size / period; i++) {
 633                 size_t tlen, len = period;
 634
 635                 for (j = 0; j < sgperiod; j++, k++) {
 636                         tlen = len;
 637
 638                         if (tlen > DMA_MAX_SIZE) {
 639                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 640                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 641                         }
 642
 643                         txd->sg[k].addr = addr;
 644                         txd->sg[k].len = tlen;
 645                         addr += tlen;
 646                         len -= tlen;
 647                 }
 648
 649                 WARN_ON(len != 0);
 650         }
 651
 652         WARN_ON(k != sglen);
 653
 654         txd->ddar = c->ddar;
 655         txd->size = size;
 656         txd->sglen = sglen;
 657         txd->cyclic = 1;
 658         txd->period = sgperiod;
 659
 660         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 661 }
 662
 663 static int sa11x0_dma_device_config(struct dma_chan *chan,
 664                                     struct dma_slave_config *cfg)
 665 {
 666         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 667         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 668         dma_addr_t addr;
 669         enum dma_slave_buswidth width;
 670         u32 maxburst;
 671
 672         if (ddar & DDAR_RW) {
 673                 addr = cfg->src_addr;
 674                 width = cfg->src_addr_width;
 675                 maxburst = cfg->src_maxburst;
 676         } else {
 677                 addr = cfg->dst_addr;
 678                 width = cfg->dst_addr_width;
 679                 maxburst = cfg->dst_maxburst;
 680         }
 681
 682         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 683              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 684             (maxburst != 4 && maxburst != 8))
 685                 return -EINVAL;
 686
 687         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 688                 ddar |= DDAR_DW;
 689         if (maxburst == 8)
 690                 ddar |= DDAR_BS;
 691
 692         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
 693                 &c->vc, &addr, width, maxburst);
 694
 695         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 696
 697         return 0;
 698 }
 699
 700 static int sa11x0_dma_device_pause(struct dma_chan *chan)
 701 {
 702         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 703         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 704         struct sa11x0_dma_phy *p;
 705         unsigned long flags;
 706
 707         dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 708         spin_lock_irqsave(&c->vc.lock, flags);
 709         if (c->status == DMA_IN_PROGRESS) {
 710                 c->status = DMA_PAUSED;
 711
 712                 p = c->phy;
 713                 if (p) {
 714                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 715                 } else {
 716                         spin_lock(&d->lock);
 717                         list_del_init(&c->node);
 718                         spin_unlock(&d->lock);
 719                 }
 720         }
 721         spin_unlock_irqrestore(&c->vc.lock, flags);
 722
 723         return 0;
 724 }
 725
 726 static int sa11x0_dma_device_resume(struct dma_chan *chan)
 727 {
 728         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 729         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 730         struct sa11x0_dma_phy *p;
 731         unsigned long flags;
 732
 733         dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 734         spin_lock_irqsave(&c->vc.lock, flags);
 735         if (c->status == DMA_PAUSED) {
 736                 c->status = DMA_IN_PROGRESS;
 737
 738                 p = c->phy;
 739                 if (p) {
 740                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 741                 } else if (!list_empty(&c->vc.desc_issued)) {
 742                         spin_lock(&d->lock);
 743                         list_add_tail(&c->node, &d->chan_pending);
 744                         spin_unlock(&d->lock);
 745                 }
 746         }
 747         spin_unlock_irqrestore(&c->vc.lock, flags);
 748
 749         return 0;
 750 }
 751
 752 static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
 753 {
 754         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 755         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 756         struct sa11x0_dma_phy *p;
 757         LIST_HEAD(head);
 758         unsigned long flags;
 759
 760         dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 761         /* Clear the tx descriptor lists */
 762         spin_lock_irqsave(&c->vc.lock, flags);
 763         vchan_get_all_descriptors(&c->vc, &head);
 764
 765         p = c->phy;
 766         if (p) {
 767                 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 768                 /* vchan is assigned to a pchan - stop the channel */
 769                 writel(DCSR_RUN | DCSR_IE |
 770                        DCSR_STRTA | DCSR_DONEA |
 771                        DCSR_STRTB | DCSR_DONEB,
 772                        p->base + DMA_DCSR_C);
 773
 774                 if (p->txd_load) {
 775                         if (p->txd_load != p->txd_done)
 776                                 list_add_tail(&p->txd_load->vd.node, &head);
 777                         p->txd_load = NULL;
 778                 }
 779                 if (p->txd_done) {
 780                         list_add_tail(&p->txd_done->vd.node, &head);
 781                         p->txd_done = NULL;
 782                 }
 783                 c->phy = NULL;
 784                 spin_lock(&d->lock);
 785                 p->vchan = NULL;
 786                 spin_unlock(&d->lock);
 787                 tasklet_schedule(&d->task);
 788         }
 789         spin_unlock_irqrestore(&c->vc.lock, flags);
 790         vchan_dma_desc_free_list(&c->vc, &head);
 791
 792         return 0;
 793 }
 794
 795 struct sa11x0_dma_channel_desc {
 796         u32 ddar;
 797         const char *name;
 798 };
 799
 800 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 801 static const struct sa11x0_dma_channel_desc chan_desc[] = {
 802         CD(Ser0UDCTr, 0),
 803         CD(Ser0UDCRc, DDAR_RW),
 804         CD(Ser1SDLCTr, 0),
 805         CD(Ser1SDLCRc, DDAR_RW),
 806         CD(Ser1UARTTr, 0),
 807         CD(Ser1UARTRc, DDAR_RW),
 808         CD(Ser2ICPTr, 0),
 809         CD(Ser2ICPRc, DDAR_RW),
 810         CD(Ser3UARTTr, 0),
 811         CD(Ser3UARTRc, DDAR_RW),
 812         CD(Ser4MCP0Tr, 0),
 813         CD(Ser4MCP0Rc, DDAR_RW),
 814         CD(Ser4MCP1Tr, 0),
 815         CD(Ser4MCP1Rc, DDAR_RW),
 816         CD(Ser4SSPTr, 0),
 817         CD(Ser4SSPRc, DDAR_RW),
 818 };
 819
 820 static const struct dma_slave_map sa11x0_dma_map[] = {
 821         { "sa11x0-ir", "tx", "Ser2ICPTr" },
 822         { "sa11x0-ir", "rx", "Ser2ICPRc" },
 823         { "sa11x0-ssp", "tx", "Ser4SSPTr" },
 824         { "sa11x0-ssp", "rx", "Ser4SSPRc" },
 825 };
 826
 827 static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
 828 {
 829         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 830         const char *p = param;
 831
 832         return !strcmp(c->name, p);
 833 }
 834
 835 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 836         struct device *dev)
 837 {
 838         unsigned i;
 839
 840         INIT_LIST_HEAD(&dmadev->channels);
 841         dmadev->dev = dev;
 842         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 843         dmadev->device_config = sa11x0_dma_device_config;
 844         dmadev->device_pause = sa11x0_dma_device_pause;
 845         dmadev->device_resume = sa11x0_dma_device_resume;
 846         dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
 847         dmadev->device_tx_status = sa11x0_dma_tx_status;
 848         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 849
 850         for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
 851                 struct sa11x0_dma_chan *c;
 852
 853                 c = kzalloc(sizeof(*c), GFP_KERNEL);
 854                 if (!c) {
 855                         dev_err(dev, "no memory for channel %u\n", i);
 856                         return -ENOMEM;
 857                 }
 858
 859                 c->status = DMA_IN_PROGRESS;
 860                 c->ddar = chan_desc[i].ddar;
 861                 c->name = chan_desc[i].name;
 862                 INIT_LIST_HEAD(&c->node);
 863
 864                 c->vc.desc_free = sa11x0_dma_free_desc;
 865                 vchan_init(&c->vc, dmadev);
 866         }
 867
 868         return dma_async_device_register(dmadev);
 869 }
 870
 871 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 872         void *data)
 873 {
 874         int irq = platform_get_irq(pdev, nr);
 875
 876         if (irq <= 0)
 877                 return -ENXIO;
 878
 879         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 880 }
 881
 882 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 883         void *data)
 884 {
 885         int irq = platform_get_irq(pdev, nr);
 886         if (irq > 0)
 887                 free_irq(irq, data);
 888 }
 889
 890 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 891 {
 892         struct sa11x0_dma_chan *c, *cn;
 893
 894         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 895                 list_del(&c->vc.chan.device_node);
 896                 tasklet_kill(&c->vc.task);
 897                 kfree(c);
 898         }
 899 }
 900
 901 static int sa11x0_dma_probe(struct platform_device *pdev)
 902 {
 903         struct sa11x0_dma_dev *d;
 904         struct resource *res;
 905         unsigned i;
 906         int ret;
 907
 908         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 909         if (!res)
 910                 return -ENXIO;
 911
 912         d = kzalloc(sizeof(*d), GFP_KERNEL);
 913         if (!d) {
 914                 ret = -ENOMEM;
 915                 goto err_alloc;
 916         }
 917
 918         spin_lock_init(&d->lock);
 919         INIT_LIST_HEAD(&d->chan_pending);
 920
 921         d->slave.filter.fn = sa11x0_dma_filter_fn;
 922         d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
 923         d->slave.filter.map = sa11x0_dma_map;
 924
 925         d->base = ioremap(res->start, resource_size(res));
 926         if (!d->base) {
 927                 ret = -ENOMEM;
 928                 goto err_ioremap;
 929         }
 930
 931         tasklet_setup(&d->task, sa11x0_dma_tasklet);
 932
 933         for (i = 0; i < NR_PHY_CHAN; i++) {
 934                 struct sa11x0_dma_phy *p = &d->phy[i];
 935
 936                 p->dev = d;
 937                 p->num = i;
 938                 p->base = d->base + i * DMA_SIZE;
 939                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 940                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 941                         p->base + DMA_DCSR_C);
 942                 writel_relaxed(0, p->base + DMA_DDAR);
 943
 944                 ret = sa11x0_dma_request_irq(pdev, i, p);
 945                 if (ret) {
 946                         while (i) {
 947                                 i--;
 948                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 949                         }
 950                         goto err_irq;
 951                 }
 952         }
 953
 954         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 955         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 956         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 957         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 958         d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 959         d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 960         d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 961                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 962         d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 963                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 964         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 965         if (ret) {
 966                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 967                         ret);
 968                 goto err_slave_reg;
 969         }
 970
 971         platform_set_drvdata(pdev, d);
 972         return 0;
 973
 974  err_slave_reg:
 975         sa11x0_dma_free_channels(&d->slave);
 976         for (i = 0; i < NR_PHY_CHAN; i++)
 977                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 978  err_irq:
 979         tasklet_kill(&d->task);
 980         iounmap(d->base);
 981  err_ioremap:
 982         kfree(d);
 983  err_alloc:
 984         return ret;
 985 }
 986
 987 static int sa11x0_dma_remove(struct platform_device *pdev)
 988 {
 989         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 990         unsigned pch;
 991
 992         dma_async_device_unregister(&d->slave);
 993
 994         sa11x0_dma_free_channels(&d->slave);
 995         for (pch = 0; pch < NR_PHY_CHAN; pch++)
 996                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 997         tasklet_kill(&d->task);
 998         iounmap(d->base);
 999         kfree(d);
1000
1001         return 0;
1002 }
1003
1004 static int sa11x0_dma_suspend(struct device *dev)
1005 {
1006         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1007         unsigned pch;
1008
1009         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1010                 struct sa11x0_dma_phy *p = &d->phy[pch];
1011                 u32 dcsr, saved_dcsr;
1012
1013                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1014                 if (dcsr & DCSR_RUN) {
1015                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1016                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1017                 }
1018
1019                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1020                 if (dcsr & DCSR_BIU) {
1021                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1022                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1023                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1024                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1025                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1026                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1027                 } else {
1028                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1029                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1030                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1031                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1032                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1033                 }
1034                 p->dcsr = saved_dcsr;
1035
1036                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1037         }
1038
1039         return 0;
1040 }
1041
1042 static int sa11x0_dma_resume(struct device *dev)
1043 {
1044         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1045         unsigned pch;
1046
1047         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1048                 struct sa11x0_dma_phy *p = &d->phy[pch];
1049                 struct sa11x0_dma_desc *txd = NULL;
1050                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1051
1052                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1053
1054                 if (p->txd_done)
1055                         txd = p->txd_done;
1056                 else if (p->txd_load)
1057                         txd = p->txd_load;
1058
1059                 if (!txd)
1060                         continue;
1061
1062                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1063
1064                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1065                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1066                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1067                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1068                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1069         }
1070
1071         return 0;
1072 }
1073
1074 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1075         .suspend_noirq = sa11x0_dma_suspend,
1076         .resume_noirq = sa11x0_dma_resume,
1077         .freeze_noirq = sa11x0_dma_suspend,
1078         .thaw_noirq = sa11x0_dma_resume,
1079         .poweroff_noirq = sa11x0_dma_suspend,
1080         .restore_noirq = sa11x0_dma_resume,
1081 };
1082
1083 static struct platform_driver sa11x0_dma_driver = {
1084         .driver = {
1085                 .name   = "sa11x0-dma",
1086                 .pm     = &sa11x0_dma_pm_ops,
1087         },
1088         .probe          = sa11x0_dma_probe,
1089         .remove         = sa11x0_dma_remove,
1090 };
1091
1092 static int __init sa11x0_dma_init(void)
1093 {
1094         return platform_driver_register(&sa11x0_dma_driver);
1095 }
1096 subsys_initcall(sa11x0_dma_init);
1097
1098 static void __exit sa11x0_dma_exit(void)
1099 {
1100         platform_driver_unregister(&sa11x0_dma_driver);
1101 }
1102 module_exit(sa11x0_dma_exit);
1103
1104 MODULE_AUTHOR("Russell King");
1105 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1106 MODULE_LICENSE("GPL v2");
1107 MODULE_ALIAS("platform:sa11x0-dma");