Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input
[platform/adaptation/renesas_rcar/renesas_kernel.git] / drivers / dma / sa11x0-dma.c
1 /*
2  * SA11x0 DMAengine support
3  *
4  * Copyright (C) 2012 Russell King
5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  */
12 #include <linux/sched.h>
13 #include <linux/device.h>
14 #include <linux/dmaengine.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/platform_device.h>
20 #include <linux/sa11x0-dma.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23
24 #include "virt-dma.h"
25
26 #define NR_PHY_CHAN     6
27 #define DMA_ALIGN       3
28 #define DMA_MAX_SIZE    0x1fff
29 #define DMA_CHUNK_SIZE  0x1000
30
31 #define DMA_DDAR        0x00
32 #define DMA_DCSR_S      0x04
33 #define DMA_DCSR_C      0x08
34 #define DMA_DCSR_R      0x0c
35 #define DMA_DBSA        0x10
36 #define DMA_DBTA        0x14
37 #define DMA_DBSB        0x18
38 #define DMA_DBTB        0x1c
39 #define DMA_SIZE        0x20
40
41 #define DCSR_RUN        (1 << 0)
42 #define DCSR_IE         (1 << 1)
43 #define DCSR_ERROR      (1 << 2)
44 #define DCSR_DONEA      (1 << 3)
45 #define DCSR_STRTA      (1 << 4)
46 #define DCSR_DONEB      (1 << 5)
47 #define DCSR_STRTB      (1 << 6)
48 #define DCSR_BIU        (1 << 7)
49
50 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
51 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
52 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
53 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
54 #define DDAR_Ser0UDCTr  (0x0 << 4)
55 #define DDAR_Ser0UDCRc  (0x1 << 4)
56 #define DDAR_Ser1SDLCTr (0x2 << 4)
57 #define DDAR_Ser1SDLCRc (0x3 << 4)
58 #define DDAR_Ser1UARTTr (0x4 << 4)
59 #define DDAR_Ser1UARTRc (0x5 << 4)
60 #define DDAR_Ser2ICPTr  (0x6 << 4)
61 #define DDAR_Ser2ICPRc  (0x7 << 4)
62 #define DDAR_Ser3UARTTr (0x8 << 4)
63 #define DDAR_Ser3UARTRc (0x9 << 4)
64 #define DDAR_Ser4MCP0Tr (0xa << 4)
65 #define DDAR_Ser4MCP0Rc (0xb << 4)
66 #define DDAR_Ser4MCP1Tr (0xc << 4)
67 #define DDAR_Ser4MCP1Rc (0xd << 4)
68 #define DDAR_Ser4SSPTr  (0xe << 4)
69 #define DDAR_Ser4SSPRc  (0xf << 4)
70
71 struct sa11x0_dma_sg {
72         u32                     addr;
73         u32                     len;
74 };
75
76 struct sa11x0_dma_desc {
77         struct virt_dma_desc    vd;
78
79         u32                     ddar;
80         size_t                  size;
81         unsigned                period;
82         bool                    cyclic;
83
84         unsigned                sglen;
85         struct sa11x0_dma_sg    sg[0];
86 };
87
88 struct sa11x0_dma_phy;
89
90 struct sa11x0_dma_chan {
91         struct virt_dma_chan    vc;
92
93         /* protected by c->vc.lock */
94         struct sa11x0_dma_phy   *phy;
95         enum dma_status         status;
96
97         /* protected by d->lock */
98         struct list_head        node;
99
100         u32                     ddar;
101         const char              *name;
102 };
103
104 struct sa11x0_dma_phy {
105         void __iomem            *base;
106         struct sa11x0_dma_dev   *dev;
107         unsigned                num;
108
109         struct sa11x0_dma_chan  *vchan;
110
111         /* Protected by c->vc.lock */
112         unsigned                sg_load;
113         struct sa11x0_dma_desc  *txd_load;
114         unsigned                sg_done;
115         struct sa11x0_dma_desc  *txd_done;
116 #ifdef CONFIG_PM_SLEEP
117         u32                     dbs[2];
118         u32                     dbt[2];
119         u32                     dcsr;
120 #endif
121 };
122
123 struct sa11x0_dma_dev {
124         struct dma_device       slave;
125         void __iomem            *base;
126         spinlock_t              lock;
127         struct tasklet_struct   task;
128         struct list_head        chan_pending;
129         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
130 };
131
132 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
133 {
134         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
135 }
136
137 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
138 {
139         return container_of(dmadev, struct sa11x0_dma_dev, slave);
140 }
141
142 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
143 {
144         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
145
146         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
147 }
148
149 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
150 {
151         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
152 }
153
154 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
155 {
156         list_del(&txd->vd.node);
157         p->txd_load = txd;
158         p->sg_load = 0;
159
160         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
161                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
162 }
163
164 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
165         struct sa11x0_dma_chan *c)
166 {
167         struct sa11x0_dma_desc *txd = p->txd_load;
168         struct sa11x0_dma_sg *sg;
169         void __iomem *base = p->base;
170         unsigned dbsx, dbtx;
171         u32 dcsr;
172
173         if (!txd)
174                 return;
175
176         dcsr = readl_relaxed(base + DMA_DCSR_R);
177
178         /* Don't try to load the next transfer if both buffers are started */
179         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
180                 return;
181
182         if (p->sg_load == txd->sglen) {
183                 if (!txd->cyclic) {
184                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
185
186                         /*
187                          * We have reached the end of the current descriptor.
188                          * Peek at the next descriptor, and if compatible with
189                          * the current, start processing it.
190                          */
191                         if (txn && txn->ddar == txd->ddar) {
192                                 txd = txn;
193                                 sa11x0_dma_start_desc(p, txn);
194                         } else {
195                                 p->txd_load = NULL;
196                                 return;
197                         }
198                 } else {
199                         /* Cyclic: reset back to beginning */
200                         p->sg_load = 0;
201                 }
202         }
203
204         sg = &txd->sg[p->sg_load++];
205
206         /* Select buffer to load according to channel status */
207         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
208             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
209                 dbsx = DMA_DBSA;
210                 dbtx = DMA_DBTA;
211                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
212         } else {
213                 dbsx = DMA_DBSB;
214                 dbtx = DMA_DBTB;
215                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
216         }
217
218         writel_relaxed(sg->addr, base + dbsx);
219         writel_relaxed(sg->len, base + dbtx);
220         writel(dcsr, base + DMA_DCSR_S);
221
222         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
223                 p->num, dcsr,
224                 'A' + (dbsx == DMA_DBSB), sg->addr,
225                 'A' + (dbtx == DMA_DBTB), sg->len);
226 }
227
228 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
229         struct sa11x0_dma_chan *c)
230 {
231         struct sa11x0_dma_desc *txd = p->txd_done;
232
233         if (++p->sg_done == txd->sglen) {
234                 if (!txd->cyclic) {
235                         vchan_cookie_complete(&txd->vd);
236
237                         p->sg_done = 0;
238                         p->txd_done = p->txd_load;
239
240                         if (!p->txd_done)
241                                 tasklet_schedule(&p->dev->task);
242                 } else {
243                         if ((p->sg_done % txd->period) == 0)
244                                 vchan_cyclic_callback(&txd->vd);
245
246                         /* Cyclic: reset back to beginning */
247                         p->sg_done = 0;
248                 }
249         }
250
251         sa11x0_dma_start_sg(p, c);
252 }
253
254 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
255 {
256         struct sa11x0_dma_phy *p = dev_id;
257         struct sa11x0_dma_dev *d = p->dev;
258         struct sa11x0_dma_chan *c;
259         u32 dcsr;
260
261         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
262         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
263                 return IRQ_NONE;
264
265         /* Clear reported status bits */
266         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
267                 p->base + DMA_DCSR_C);
268
269         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
270
271         if (dcsr & DCSR_ERROR) {
272                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
273                         p->num, dcsr,
274                         readl_relaxed(p->base + DMA_DDAR),
275                         readl_relaxed(p->base + DMA_DBSA),
276                         readl_relaxed(p->base + DMA_DBTA),
277                         readl_relaxed(p->base + DMA_DBSB),
278                         readl_relaxed(p->base + DMA_DBTB));
279         }
280
281         c = p->vchan;
282         if (c) {
283                 unsigned long flags;
284
285                 spin_lock_irqsave(&c->vc.lock, flags);
286                 /*
287                  * Now that we're holding the lock, check that the vchan
288                  * really is associated with this pchan before touching the
289                  * hardware.  This should always succeed, because we won't
290                  * change p->vchan or c->phy while the channel is actively
291                  * transferring.
292                  */
293                 if (c->phy == p) {
294                         if (dcsr & DCSR_DONEA)
295                                 sa11x0_dma_complete(p, c);
296                         if (dcsr & DCSR_DONEB)
297                                 sa11x0_dma_complete(p, c);
298                 }
299                 spin_unlock_irqrestore(&c->vc.lock, flags);
300         }
301
302         return IRQ_HANDLED;
303 }
304
305 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
306 {
307         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
308
309         /* If the issued list is empty, we have no further txds to process */
310         if (txd) {
311                 struct sa11x0_dma_phy *p = c->phy;
312
313                 sa11x0_dma_start_desc(p, txd);
314                 p->txd_done = txd;
315                 p->sg_done = 0;
316
317                 /* The channel should not have any transfers started */
318                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
319                                       (DCSR_STRTA | DCSR_STRTB));
320
321                 /* Clear the run and start bits before changing DDAR */
322                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
323                                p->base + DMA_DCSR_C);
324                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
325
326                 /* Try to start both buffers */
327                 sa11x0_dma_start_sg(p, c);
328                 sa11x0_dma_start_sg(p, c);
329         }
330 }
331
332 static void sa11x0_dma_tasklet(unsigned long arg)
333 {
334         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
335         struct sa11x0_dma_phy *p;
336         struct sa11x0_dma_chan *c;
337         unsigned pch, pch_alloc = 0;
338
339         dev_dbg(d->slave.dev, "tasklet enter\n");
340
341         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
342                 spin_lock_irq(&c->vc.lock);
343                 p = c->phy;
344                 if (p && !p->txd_done) {
345                         sa11x0_dma_start_txd(c);
346                         if (!p->txd_done) {
347                                 /* No current txd associated with this channel */
348                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
349
350                                 /* Mark this channel free */
351                                 c->phy = NULL;
352                                 p->vchan = NULL;
353                         }
354                 }
355                 spin_unlock_irq(&c->vc.lock);
356         }
357
358         spin_lock_irq(&d->lock);
359         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
360                 p = &d->phy[pch];
361
362                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
363                         c = list_first_entry(&d->chan_pending,
364                                 struct sa11x0_dma_chan, node);
365                         list_del_init(&c->node);
366
367                         pch_alloc |= 1 << pch;
368
369                         /* Mark this channel allocated */
370                         p->vchan = c;
371
372                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
373                 }
374         }
375         spin_unlock_irq(&d->lock);
376
377         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
378                 if (pch_alloc & (1 << pch)) {
379                         p = &d->phy[pch];
380                         c = p->vchan;
381
382                         spin_lock_irq(&c->vc.lock);
383                         c->phy = p;
384
385                         sa11x0_dma_start_txd(c);
386                         spin_unlock_irq(&c->vc.lock);
387                 }
388         }
389
390         dev_dbg(d->slave.dev, "tasklet exit\n");
391 }
392
393
394 static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
395 {
396         return 0;
397 }
398
399 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
400 {
401         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
402         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
403         unsigned long flags;
404
405         spin_lock_irqsave(&d->lock, flags);
406         list_del_init(&c->node);
407         spin_unlock_irqrestore(&d->lock, flags);
408
409         vchan_free_chan_resources(&c->vc);
410 }
411
412 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
413 {
414         unsigned reg;
415         u32 dcsr;
416
417         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
418
419         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
420             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
421                 reg = DMA_DBSA;
422         else
423                 reg = DMA_DBSB;
424
425         return readl_relaxed(p->base + reg);
426 }
427
428 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
429         dma_cookie_t cookie, struct dma_tx_state *state)
430 {
431         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
432         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
433         struct sa11x0_dma_phy *p;
434         struct virt_dma_desc *vd;
435         unsigned long flags;
436         enum dma_status ret;
437
438         ret = dma_cookie_status(&c->vc.chan, cookie, state);
439         if (ret == DMA_SUCCESS)
440                 return ret;
441
442         if (!state)
443                 return c->status;
444
445         spin_lock_irqsave(&c->vc.lock, flags);
446         p = c->phy;
447
448         /*
449          * If the cookie is on our issue queue, then the residue is
450          * its total size.
451          */
452         vd = vchan_find_desc(&c->vc, cookie);
453         if (vd) {
454                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
455         } else if (!p) {
456                 state->residue = 0;
457         } else {
458                 struct sa11x0_dma_desc *txd;
459                 size_t bytes = 0;
460
461                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
462                         txd = p->txd_done;
463                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
464                         txd = p->txd_load;
465                 else
466                         txd = NULL;
467
468                 ret = c->status;
469                 if (txd) {
470                         dma_addr_t addr = sa11x0_dma_pos(p);
471                         unsigned i;
472
473                         dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
474
475                         for (i = 0; i < txd->sglen; i++) {
476                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
477                                         i, txd->sg[i].addr, txd->sg[i].len);
478                                 if (addr >= txd->sg[i].addr &&
479                                     addr < txd->sg[i].addr + txd->sg[i].len) {
480                                         unsigned len;
481
482                                         len = txd->sg[i].len -
483                                                 (addr - txd->sg[i].addr);
484                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
485                                                 i, len);
486                                         bytes += len;
487                                         i++;
488                                         break;
489                                 }
490                         }
491                         for (; i < txd->sglen; i++) {
492                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
493                                         i, txd->sg[i].addr, txd->sg[i].len);
494                                 bytes += txd->sg[i].len;
495                         }
496                 }
497                 state->residue = bytes;
498         }
499         spin_unlock_irqrestore(&c->vc.lock, flags);
500
501         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
502
503         return ret;
504 }
505
506 /*
507  * Move pending txds to the issued list, and re-init pending list.
508  * If not already pending, add this channel to the list of pending
509  * channels and trigger the tasklet to run.
510  */
511 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
512 {
513         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
514         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
515         unsigned long flags;
516
517         spin_lock_irqsave(&c->vc.lock, flags);
518         if (vchan_issue_pending(&c->vc)) {
519                 if (!c->phy) {
520                         spin_lock(&d->lock);
521                         if (list_empty(&c->node)) {
522                                 list_add_tail(&c->node, &d->chan_pending);
523                                 tasklet_schedule(&d->task);
524                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
525                         }
526                         spin_unlock(&d->lock);
527                 }
528         } else
529                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
530         spin_unlock_irqrestore(&c->vc.lock, flags);
531 }
532
533 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
534         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
535         enum dma_transfer_direction dir, unsigned long flags, void *context)
536 {
537         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
538         struct sa11x0_dma_desc *txd;
539         struct scatterlist *sgent;
540         unsigned i, j = sglen;
541         size_t size = 0;
542
543         /* SA11x0 channels can only operate in their native direction */
544         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
545                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
546                         &c->vc, c->ddar, dir);
547                 return NULL;
548         }
549
550         /* Do not allow zero-sized txds */
551         if (sglen == 0)
552                 return NULL;
553
554         for_each_sg(sg, sgent, sglen, i) {
555                 dma_addr_t addr = sg_dma_address(sgent);
556                 unsigned int len = sg_dma_len(sgent);
557
558                 if (len > DMA_MAX_SIZE)
559                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
560                 if (addr & DMA_ALIGN) {
561                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
562                                 &c->vc, addr);
563                         return NULL;
564                 }
565         }
566
567         txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
568         if (!txd) {
569                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
570                 return NULL;
571         }
572
573         j = 0;
574         for_each_sg(sg, sgent, sglen, i) {
575                 dma_addr_t addr = sg_dma_address(sgent);
576                 unsigned len = sg_dma_len(sgent);
577
578                 size += len;
579
580                 do {
581                         unsigned tlen = len;
582
583                         /*
584                          * Check whether the transfer will fit.  If not, try
585                          * to split the transfer up such that we end up with
586                          * equal chunks - but make sure that we preserve the
587                          * alignment.  This avoids small segments.
588                          */
589                         if (tlen > DMA_MAX_SIZE) {
590                                 unsigned mult = DIV_ROUND_UP(tlen,
591                                         DMA_MAX_SIZE & ~DMA_ALIGN);
592
593                                 tlen = (tlen / mult) & ~DMA_ALIGN;
594                         }
595
596                         txd->sg[j].addr = addr;
597                         txd->sg[j].len = tlen;
598
599                         addr += tlen;
600                         len -= tlen;
601                         j++;
602                 } while (len);
603         }
604
605         txd->ddar = c->ddar;
606         txd->size = size;
607         txd->sglen = j;
608
609         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
610                 &c->vc, &txd->vd, txd->size, txd->sglen);
611
612         return vchan_tx_prep(&c->vc, &txd->vd, flags);
613 }
614
615 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
616         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
617         enum dma_transfer_direction dir, unsigned long flags, void *context)
618 {
619         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
620         struct sa11x0_dma_desc *txd;
621         unsigned i, j, k, sglen, sgperiod;
622
623         /* SA11x0 channels can only operate in their native direction */
624         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
625                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
626                         &c->vc, c->ddar, dir);
627                 return NULL;
628         }
629
630         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
631         sglen = size * sgperiod / period;
632
633         /* Do not allow zero-sized txds */
634         if (sglen == 0)
635                 return NULL;
636
637         txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC);
638         if (!txd) {
639                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
640                 return NULL;
641         }
642
643         for (i = k = 0; i < size / period; i++) {
644                 size_t tlen, len = period;
645
646                 for (j = 0; j < sgperiod; j++, k++) {
647                         tlen = len;
648
649                         if (tlen > DMA_MAX_SIZE) {
650                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
651                                 tlen = (tlen / mult) & ~DMA_ALIGN;
652                         }
653
654                         txd->sg[k].addr = addr;
655                         txd->sg[k].len = tlen;
656                         addr += tlen;
657                         len -= tlen;
658                 }
659
660                 WARN_ON(len != 0);
661         }
662
663         WARN_ON(k != sglen);
664
665         txd->ddar = c->ddar;
666         txd->size = size;
667         txd->sglen = sglen;
668         txd->cyclic = 1;
669         txd->period = sgperiod;
670
671         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
672 }
673
674 static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg)
675 {
676         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
677         dma_addr_t addr;
678         enum dma_slave_buswidth width;
679         u32 maxburst;
680
681         if (ddar & DDAR_RW) {
682                 addr = cfg->src_addr;
683                 width = cfg->src_addr_width;
684                 maxburst = cfg->src_maxburst;
685         } else {
686                 addr = cfg->dst_addr;
687                 width = cfg->dst_addr_width;
688                 maxburst = cfg->dst_maxburst;
689         }
690
691         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
692              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
693             (maxburst != 4 && maxburst != 8))
694                 return -EINVAL;
695
696         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
697                 ddar |= DDAR_DW;
698         if (maxburst == 8)
699                 ddar |= DDAR_BS;
700
701         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
702                 &c->vc, addr, width, maxburst);
703
704         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
705
706         return 0;
707 }
708
709 static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
710         unsigned long arg)
711 {
712         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
713         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
714         struct sa11x0_dma_phy *p;
715         LIST_HEAD(head);
716         unsigned long flags;
717         int ret;
718
719         switch (cmd) {
720         case DMA_SLAVE_CONFIG:
721                 return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg);
722
723         case DMA_TERMINATE_ALL:
724                 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
725                 /* Clear the tx descriptor lists */
726                 spin_lock_irqsave(&c->vc.lock, flags);
727                 vchan_get_all_descriptors(&c->vc, &head);
728
729                 p = c->phy;
730                 if (p) {
731                         dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
732                         /* vchan is assigned to a pchan - stop the channel */
733                         writel(DCSR_RUN | DCSR_IE |
734                                 DCSR_STRTA | DCSR_DONEA |
735                                 DCSR_STRTB | DCSR_DONEB,
736                                 p->base + DMA_DCSR_C);
737
738                         if (p->txd_load) {
739                                 if (p->txd_load != p->txd_done)
740                                         list_add_tail(&p->txd_load->vd.node, &head);
741                                 p->txd_load = NULL;
742                         }
743                         if (p->txd_done) {
744                                 list_add_tail(&p->txd_done->vd.node, &head);
745                                 p->txd_done = NULL;
746                         }
747                         c->phy = NULL;
748                         spin_lock(&d->lock);
749                         p->vchan = NULL;
750                         spin_unlock(&d->lock);
751                         tasklet_schedule(&d->task);
752                 }
753                 spin_unlock_irqrestore(&c->vc.lock, flags);
754                 vchan_dma_desc_free_list(&c->vc, &head);
755                 ret = 0;
756                 break;
757
758         case DMA_PAUSE:
759                 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
760                 spin_lock_irqsave(&c->vc.lock, flags);
761                 if (c->status == DMA_IN_PROGRESS) {
762                         c->status = DMA_PAUSED;
763
764                         p = c->phy;
765                         if (p) {
766                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
767                         } else {
768                                 spin_lock(&d->lock);
769                                 list_del_init(&c->node);
770                                 spin_unlock(&d->lock);
771                         }
772                 }
773                 spin_unlock_irqrestore(&c->vc.lock, flags);
774                 ret = 0;
775                 break;
776
777         case DMA_RESUME:
778                 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
779                 spin_lock_irqsave(&c->vc.lock, flags);
780                 if (c->status == DMA_PAUSED) {
781                         c->status = DMA_IN_PROGRESS;
782
783                         p = c->phy;
784                         if (p) {
785                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
786                         } else if (!list_empty(&c->vc.desc_issued)) {
787                                 spin_lock(&d->lock);
788                                 list_add_tail(&c->node, &d->chan_pending);
789                                 spin_unlock(&d->lock);
790                         }
791                 }
792                 spin_unlock_irqrestore(&c->vc.lock, flags);
793                 ret = 0;
794                 break;
795
796         default:
797                 ret = -ENXIO;
798                 break;
799         }
800
801         return ret;
802 }
803
804 struct sa11x0_dma_channel_desc {
805         u32 ddar;
806         const char *name;
807 };
808
809 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
810 static const struct sa11x0_dma_channel_desc chan_desc[] = {
811         CD(Ser0UDCTr, 0),
812         CD(Ser0UDCRc, DDAR_RW),
813         CD(Ser1SDLCTr, 0),
814         CD(Ser1SDLCRc, DDAR_RW),
815         CD(Ser1UARTTr, 0),
816         CD(Ser1UARTRc, DDAR_RW),
817         CD(Ser2ICPTr, 0),
818         CD(Ser2ICPRc, DDAR_RW),
819         CD(Ser3UARTTr, 0),
820         CD(Ser3UARTRc, DDAR_RW),
821         CD(Ser4MCP0Tr, 0),
822         CD(Ser4MCP0Rc, DDAR_RW),
823         CD(Ser4MCP1Tr, 0),
824         CD(Ser4MCP1Rc, DDAR_RW),
825         CD(Ser4SSPTr, 0),
826         CD(Ser4SSPRc, DDAR_RW),
827 };
828
829 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
830         struct device *dev)
831 {
832         unsigned i;
833
834         dmadev->chancnt = ARRAY_SIZE(chan_desc);
835         INIT_LIST_HEAD(&dmadev->channels);
836         dmadev->dev = dev;
837         dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
838         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
839         dmadev->device_control = sa11x0_dma_control;
840         dmadev->device_tx_status = sa11x0_dma_tx_status;
841         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
842
843         for (i = 0; i < dmadev->chancnt; i++) {
844                 struct sa11x0_dma_chan *c;
845
846                 c = kzalloc(sizeof(*c), GFP_KERNEL);
847                 if (!c) {
848                         dev_err(dev, "no memory for channel %u\n", i);
849                         return -ENOMEM;
850                 }
851
852                 c->status = DMA_IN_PROGRESS;
853                 c->ddar = chan_desc[i].ddar;
854                 c->name = chan_desc[i].name;
855                 INIT_LIST_HEAD(&c->node);
856
857                 c->vc.desc_free = sa11x0_dma_free_desc;
858                 vchan_init(&c->vc, dmadev);
859         }
860
861         return dma_async_device_register(dmadev);
862 }
863
864 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
865         void *data)
866 {
867         int irq = platform_get_irq(pdev, nr);
868
869         if (irq <= 0)
870                 return -ENXIO;
871
872         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
873 }
874
875 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
876         void *data)
877 {
878         int irq = platform_get_irq(pdev, nr);
879         if (irq > 0)
880                 free_irq(irq, data);
881 }
882
883 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
884 {
885         struct sa11x0_dma_chan *c, *cn;
886
887         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
888                 list_del(&c->vc.chan.device_node);
889                 tasklet_kill(&c->vc.task);
890                 kfree(c);
891         }
892 }
893
894 static int sa11x0_dma_probe(struct platform_device *pdev)
895 {
896         struct sa11x0_dma_dev *d;
897         struct resource *res;
898         unsigned i;
899         int ret;
900
901         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
902         if (!res)
903                 return -ENXIO;
904
905         d = kzalloc(sizeof(*d), GFP_KERNEL);
906         if (!d) {
907                 ret = -ENOMEM;
908                 goto err_alloc;
909         }
910
911         spin_lock_init(&d->lock);
912         INIT_LIST_HEAD(&d->chan_pending);
913
914         d->base = ioremap(res->start, resource_size(res));
915         if (!d->base) {
916                 ret = -ENOMEM;
917                 goto err_ioremap;
918         }
919
920         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
921
922         for (i = 0; i < NR_PHY_CHAN; i++) {
923                 struct sa11x0_dma_phy *p = &d->phy[i];
924
925                 p->dev = d;
926                 p->num = i;
927                 p->base = d->base + i * DMA_SIZE;
928                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
929                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
930                         p->base + DMA_DCSR_C);
931                 writel_relaxed(0, p->base + DMA_DDAR);
932
933                 ret = sa11x0_dma_request_irq(pdev, i, p);
934                 if (ret) {
935                         while (i) {
936                                 i--;
937                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
938                         }
939                         goto err_irq;
940                 }
941         }
942
943         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
944         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
945         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
946         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
947         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
948         if (ret) {
949                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
950                         ret);
951                 goto err_slave_reg;
952         }
953
954         platform_set_drvdata(pdev, d);
955         return 0;
956
957  err_slave_reg:
958         sa11x0_dma_free_channels(&d->slave);
959         for (i = 0; i < NR_PHY_CHAN; i++)
960                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
961  err_irq:
962         tasklet_kill(&d->task);
963         iounmap(d->base);
964  err_ioremap:
965         kfree(d);
966  err_alloc:
967         return ret;
968 }
969
970 static int sa11x0_dma_remove(struct platform_device *pdev)
971 {
972         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
973         unsigned pch;
974
975         dma_async_device_unregister(&d->slave);
976
977         sa11x0_dma_free_channels(&d->slave);
978         for (pch = 0; pch < NR_PHY_CHAN; pch++)
979                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
980         tasklet_kill(&d->task);
981         iounmap(d->base);
982         kfree(d);
983
984         return 0;
985 }
986
987 #ifdef CONFIG_PM_SLEEP
988 static int sa11x0_dma_suspend(struct device *dev)
989 {
990         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
991         unsigned pch;
992
993         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
994                 struct sa11x0_dma_phy *p = &d->phy[pch];
995                 u32 dcsr, saved_dcsr;
996
997                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
998                 if (dcsr & DCSR_RUN) {
999                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1000                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1001                 }
1002
1003                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1004                 if (dcsr & DCSR_BIU) {
1005                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1006                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1007                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1008                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1009                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1010                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1011                 } else {
1012                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1013                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1014                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1015                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1016                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1017                 }
1018                 p->dcsr = saved_dcsr;
1019
1020                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1021         }
1022
1023         return 0;
1024 }
1025
1026 static int sa11x0_dma_resume(struct device *dev)
1027 {
1028         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1029         unsigned pch;
1030
1031         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1032                 struct sa11x0_dma_phy *p = &d->phy[pch];
1033                 struct sa11x0_dma_desc *txd = NULL;
1034                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1035
1036                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1037
1038                 if (p->txd_done)
1039                         txd = p->txd_done;
1040                 else if (p->txd_load)
1041                         txd = p->txd_load;
1042
1043                 if (!txd)
1044                         continue;
1045
1046                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1047
1048                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1049                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1050                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1051                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1052                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1053         }
1054
1055         return 0;
1056 }
1057 #endif
1058
1059 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1060         .suspend_noirq = sa11x0_dma_suspend,
1061         .resume_noirq = sa11x0_dma_resume,
1062         .freeze_noirq = sa11x0_dma_suspend,
1063         .thaw_noirq = sa11x0_dma_resume,
1064         .poweroff_noirq = sa11x0_dma_suspend,
1065         .restore_noirq = sa11x0_dma_resume,
1066 };
1067
1068 static struct platform_driver sa11x0_dma_driver = {
1069         .driver = {
1070                 .name   = "sa11x0-dma",
1071                 .owner  = THIS_MODULE,
1072                 .pm     = &sa11x0_dma_pm_ops,
1073         },
1074         .probe          = sa11x0_dma_probe,
1075         .remove         = sa11x0_dma_remove,
1076 };
1077
1078 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1079 {
1080         if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1081                 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1082                 const char *p = param;
1083
1084                 return !strcmp(c->name, p);
1085         }
1086         return false;
1087 }
1088 EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1089
1090 static int __init sa11x0_dma_init(void)
1091 {
1092         return platform_driver_register(&sa11x0_dma_driver);
1093 }
1094 subsys_initcall(sa11x0_dma_init);
1095
1096 static void __exit sa11x0_dma_exit(void)
1097 {
1098         platform_driver_unregister(&sa11x0_dma_driver);
1099 }
1100 module_exit(sa11x0_dma_exit);
1101
1102 MODULE_AUTHOR("Russell King");
1103 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1104 MODULE_LICENSE("GPL v2");
1105 MODULE_ALIAS("platform:sa11x0-dma");