drivers/dma/ste_dma40.c

   1 /*
   2  * Copyright (C) Ericsson AB 2007-2008
   3  * Copyright (C) ST-Ericsson SA 2008-2010
   4  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
   5  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
   6  * License terms: GNU General Public License (GPL) version 2
   7  */
   8
   9 #include <linux/kernel.h>
  10 #include <linux/slab.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/clk.h>
  14 #include <linux/delay.h>
  15 #include <linux/err.h>
  16
  17 #include <plat/ste_dma40.h>
  18
  19 #include "ste_dma40_ll.h"
  20
  21 #define D40_NAME "dma40"
  22
  23 #define D40_PHY_CHAN -1
  24
  25 /* For masking out/in 2 bit channel positions */
  26 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
  27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  28
  29 /* Maximum iterations taken before giving up suspending a channel */
  30 #define D40_SUSPEND_MAX_IT 500
  31
  32 /* Hardware requirement on LCLA alignment */
  33 #define LCLA_ALIGNMENT 0x40000
  34
  35 /* Max number of links per event group */
  36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
  37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  38
  39 /* Attempts before giving up to trying to get pages that are aligned */
  40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
  41
  42 /* Bit markings for allocation map */
  43 #define D40_ALLOC_FREE          (1 << 31)
  44 #define D40_ALLOC_PHY           (1 << 30)
  45 #define D40_ALLOC_LOG_FREE      0
  46
  47 /* Hardware designer of the block */
  48 #define D40_HW_DESIGNER 0x8
  49
  50 /**
  51  * enum 40_command - The different commands and/or statuses.
  52  *
  53  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  54  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  55  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  56  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  57  */
  58 enum d40_command {
  59         D40_DMA_STOP            = 0,
  60         D40_DMA_RUN             = 1,
  61         D40_DMA_SUSPEND_REQ     = 2,
  62         D40_DMA_SUSPENDED       = 3
  63 };
  64
  65 /**
  66  * struct d40_lli_pool - Structure for keeping LLIs in memory
  67  *
  68  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  69  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  70  * pre_alloc_lli is used.
  71  * @dma_addr: DMA address, if mapped
  72  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  73  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  74  * one buffer to one buffer.
  75  */
  76 struct d40_lli_pool {
  77         void    *base;
  78         int      size;
  79         dma_addr_t      dma_addr;
  80         /* Space for dst and src, plus an extra for padding */
  81         u8       pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  82 };
  83
  84 /**
  85  * struct d40_desc - A descriptor is one DMA job.
  86  *
  87  * @lli_phy: LLI settings for physical channel. Both src and dst=
  88  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  89  * lli_len equals one.
  90  * @lli_log: Same as above but for logical channels.
  91  * @lli_pool: The pool with two entries pre-allocated.
  92  * @lli_len: Number of llis of current descriptor.
  93  * @lli_current: Number of transfered llis.
  94  * @lcla_alloc: Number of LCLA entries allocated.
  95  * @txd: DMA engine struct. Used for among other things for communication
  96  * during a transfer.
  97  * @node: List entry.
  98  * @is_in_client_list: true if the client owns this descriptor.
  99  * the previous one.
 100  *
 101  * This descriptor is used for both logical and physical transfers.
 102  */
 103 struct d40_desc {
 104         /* LLI physical */
 105         struct d40_phy_lli_bidir         lli_phy;
 106         /* LLI logical */
 107         struct d40_log_lli_bidir         lli_log;
 108
 109         struct d40_lli_pool              lli_pool;
 110         int                              lli_len;
 111         int                              lli_current;
 112         int                              lcla_alloc;
 113
 114         struct dma_async_tx_descriptor   txd;
 115         struct list_head                 node;
 116
 117         bool                             is_in_client_list;
 118 };
 119
 120 /**
 121  * struct d40_lcla_pool - LCLA pool settings and data.
 122  *
 123  * @base: The virtual address of LCLA. 18 bit aligned.
 124  * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
 125  * This pointer is only there for clean-up on error.
 126  * @pages: The number of pages needed for all physical channels.
 127  * Only used later for clean-up on error
 128  * @lock: Lock to protect the content in this struct.
 129  * @alloc_map: big map over which LCLA entry is own by which job.
 130  */
 131 struct d40_lcla_pool {
 132         void            *base;
 133         dma_addr_t      dma_addr;
 134         void            *base_unaligned;
 135         int              pages;
 136         spinlock_t       lock;
 137         struct d40_desc **alloc_map;
 138 };
 139
 140 /**
 141  * struct d40_phy_res - struct for handling eventlines mapped to physical
 142  * channels.
 143  *
 144  * @lock: A lock protection this entity.
 145  * @num: The physical channel number of this entity.
 146  * @allocated_src: Bit mapped to show which src event line's are mapped to
 147  * this physical channel. Can also be free or physically allocated.
 148  * @allocated_dst: Same as for src but is dst.
 149  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
 150  * event line number.
 151  */
 152 struct d40_phy_res {
 153         spinlock_t lock;
 154         int        num;
 155         u32        allocated_src;
 156         u32        allocated_dst;
 157 };
 158
 159 struct d40_base;
 160
 161 /**
 162  * struct d40_chan - Struct that describes a channel.
 163  *
 164  * @lock: A spinlock to protect this struct.
 165  * @log_num: The logical number, if any of this channel.
 166  * @completed: Starts with 1, after first interrupt it is set to dma engine's
 167  * current cookie.
 168  * @pending_tx: The number of pending transfers. Used between interrupt handler
 169  * and tasklet.
 170  * @busy: Set to true when transfer is ongoing on this channel.
 171  * @phy_chan: Pointer to physical channel which this instance runs on. If this
 172  * point is NULL, then the channel is not allocated.
 173  * @chan: DMA engine handle.
 174  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
 175  * transfer and call client callback.
 176  * @client: Cliented owned descriptor list.
 177  * @active: Active descriptor.
 178  * @queue: Queued jobs.
 179  * @dma_cfg: The client configuration of this dma channel.
 180  * @configured: whether the dma_cfg configuration is valid
 181  * @base: Pointer to the device instance struct.
 182  * @src_def_cfg: Default cfg register setting for src.
 183  * @dst_def_cfg: Default cfg register setting for dst.
 184  * @log_def: Default logical channel settings.
 185  * @lcla: Space for one dst src pair for logical channel transfers.
 186  * @lcpa: Pointer to dst and src lcpa settings.
 187  *
 188  * This struct can either "be" a logical or a physical channel.
 189  */
 190 struct d40_chan {
 191         spinlock_t                       lock;
 192         int                              log_num;
 193         /* ID of the most recent completed transfer */
 194         int                              completed;
 195         int                              pending_tx;
 196         bool                             busy;
 197         struct d40_phy_res              *phy_chan;
 198         struct dma_chan                  chan;
 199         struct tasklet_struct            tasklet;
 200         struct list_head                 client;
 201         struct list_head                 active;
 202         struct list_head                 queue;
 203         struct stedma40_chan_cfg         dma_cfg;
 204         bool                             configured;
 205         struct d40_base                 *base;
 206         /* Default register configurations */
 207         u32                              src_def_cfg;
 208         u32                              dst_def_cfg;
 209         struct d40_def_lcsp              log_def;
 210         struct d40_log_lli_full         *lcpa;
 211         /* Runtime reconfiguration */
 212         dma_addr_t                      runtime_addr;
 213         enum dma_data_direction         runtime_direction;
 214 };
 215
 216 /**
 217  * struct d40_base - The big global struct, one for each probe'd instance.
 218  *
 219  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
 220  * @execmd_lock: Lock for execute command usage since several channels share
 221  * the same physical register.
 222  * @dev: The device structure.
 223  * @virtbase: The virtual base address of the DMA's register.
 224  * @rev: silicon revision detected.
 225  * @clk: Pointer to the DMA clock structure.
 226  * @phy_start: Physical memory start of the DMA registers.
 227  * @phy_size: Size of the DMA register map.
 228  * @irq: The IRQ number.
 229  * @num_phy_chans: The number of physical channels. Read from HW. This
 230  * is the number of available channels for this driver, not counting "Secure
 231  * mode" allocated physical channels.
 232  * @num_log_chans: The number of logical channels. Calculated from
 233  * num_phy_chans.
 234  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
 235  * @dma_slave: dma_device channels that can do only do slave transfers.
 236  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
 237  * @log_chans: Room for all possible logical channels in system.
 238  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
 239  * to log_chans entries.
 240  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
 241  * to phy_chans entries.
 242  * @plat_data: Pointer to provided platform_data which is the driver
 243  * configuration.
 244  * @phy_res: Vector containing all physical channels.
 245  * @lcla_pool: lcla pool settings and data.
 246  * @lcpa_base: The virtual mapped address of LCPA.
 247  * @phy_lcpa: The physical address of the LCPA.
 248  * @lcpa_size: The size of the LCPA area.
 249  * @desc_slab: cache for descriptors.
 250  */
 251 struct d40_base {
 252         spinlock_t                       interrupt_lock;
 253         spinlock_t                       execmd_lock;
 254         struct device                    *dev;
 255         void __iomem                     *virtbase;
 256         u8                                rev:4;
 257         struct clk                       *clk;
 258         phys_addr_t                       phy_start;
 259         resource_size_t                   phy_size;
 260         int                               irq;
 261         int                               num_phy_chans;
 262         int                               num_log_chans;
 263         struct dma_device                 dma_both;
 264         struct dma_device                 dma_slave;
 265         struct dma_device                 dma_memcpy;
 266         struct d40_chan                  *phy_chans;
 267         struct d40_chan                  *log_chans;
 268         struct d40_chan                 **lookup_log_chans;
 269         struct d40_chan                 **lookup_phy_chans;
 270         struct stedma40_platform_data    *plat_data;
 271         /* Physical half channels */
 272         struct d40_phy_res               *phy_res;
 273         struct d40_lcla_pool              lcla_pool;
 274         void                             *lcpa_base;
 275         dma_addr_t                        phy_lcpa;
 276         resource_size_t                   lcpa_size;
 277         struct kmem_cache                *desc_slab;
 278 };
 279
 280 /**
 281  * struct d40_interrupt_lookup - lookup table for interrupt handler
 282  *
 283  * @src: Interrupt mask register.
 284  * @clr: Interrupt clear register.
 285  * @is_error: true if this is an error interrupt.
 286  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
 287  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
 288  */
 289 struct d40_interrupt_lookup {
 290         u32 src;
 291         u32 clr;
 292         bool is_error;
 293         int offset;
 294 };
 295
 296 /**
 297  * struct d40_reg_val - simple lookup struct
 298  *
 299  * @reg: The register.
 300  * @val: The value that belongs to the register in reg.
 301  */
 302 struct d40_reg_val {
 303         unsigned int reg;
 304         unsigned int val;
 305 };
 306
 307 static struct device *chan2dev(struct d40_chan *d40c)
 308 {
 309         return &d40c->chan.dev->device;
 310 }
 311
 312 static bool chan_is_physical(struct d40_chan *chan)
 313 {
 314         return chan->log_num == D40_PHY_CHAN;
 315 }
 316
 317 static bool chan_is_logical(struct d40_chan *chan)
 318 {
 319         return !chan_is_physical(chan);
 320 }
 321
 322 static void __iomem *chan_base(struct d40_chan *chan)
 323 {
 324         return chan->base->virtbase + D40_DREG_PCBASE +
 325                chan->phy_chan->num * D40_DREG_PCDELTA;
 326 }
 327
 328 #define d40_err(dev, format, arg...)            \
 329         dev_err(dev, "[%s] " format, __func__, ## arg)
 330
 331 #define chan_err(d40c, format, arg...)          \
 332         d40_err(chan2dev(d40c), format, ## arg)
 333
 334 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
 335                               int lli_len)
 336 {
 337         bool is_log = chan_is_logical(d40c);
 338         u32 align;
 339         void *base;
 340
 341         if (is_log)
 342                 align = sizeof(struct d40_log_lli);
 343         else
 344                 align = sizeof(struct d40_phy_lli);
 345
 346         if (lli_len == 1) {
 347                 base = d40d->lli_pool.pre_alloc_lli;
 348                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 349                 d40d->lli_pool.base = NULL;
 350         } else {
 351                 d40d->lli_pool.size = lli_len * 2 * align;
 352
 353                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 354                 d40d->lli_pool.base = base;
 355
 356                 if (d40d->lli_pool.base == NULL)
 357                         return -ENOMEM;
 358         }
 359
 360         if (is_log) {
 361                 d40d->lli_log.src = PTR_ALIGN(base, align);
 362                 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
 363
 364                 d40d->lli_pool.dma_addr = 0;
 365         } else {
 366                 d40d->lli_phy.src = PTR_ALIGN(base, align);
 367                 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
 368
 369                 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
 370                                                          d40d->lli_phy.src,
 371                                                          d40d->lli_pool.size,
 372                                                          DMA_TO_DEVICE);
 373
 374                 if (dma_mapping_error(d40c->base->dev,
 375                                       d40d->lli_pool.dma_addr)) {
 376                         kfree(d40d->lli_pool.base);
 377                         d40d->lli_pool.base = NULL;
 378                         d40d->lli_pool.dma_addr = 0;
 379                         return -ENOMEM;
 380                 }
 381         }
 382
 383         return 0;
 384 }
 385
 386 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
 387 {
 388         if (d40d->lli_pool.dma_addr)
 389                 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
 390                                  d40d->lli_pool.size, DMA_TO_DEVICE);
 391
 392         kfree(d40d->lli_pool.base);
 393         d40d->lli_pool.base = NULL;
 394         d40d->lli_pool.size = 0;
 395         d40d->lli_log.src = NULL;
 396         d40d->lli_log.dst = NULL;
 397         d40d->lli_phy.src = NULL;
 398         d40d->lli_phy.dst = NULL;
 399 }
 400
 401 static int d40_lcla_alloc_one(struct d40_chan *d40c,
 402                               struct d40_desc *d40d)
 403 {
 404         unsigned long flags;
 405         int i;
 406         int ret = -EINVAL;
 407         int p;
 408
 409         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 410
 411         p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
 412
 413         /*
 414          * Allocate both src and dst at the same time, therefore the half
 415          * start on 1 since 0 can't be used since zero is used as end marker.
 416          */
 417         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 418                 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
 419                         d40c->base->lcla_pool.alloc_map[p + i] = d40d;
 420                         d40d->lcla_alloc++;
 421                         ret = i;
 422                         break;
 423                 }
 424         }
 425
 426         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 427
 428         return ret;
 429 }
 430
 431 static int d40_lcla_free_all(struct d40_chan *d40c,
 432                              struct d40_desc *d40d)
 433 {
 434         unsigned long flags;
 435         int i;
 436         int ret = -EINVAL;
 437
 438         if (chan_is_physical(d40c))
 439                 return 0;
 440
 441         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 442
 443         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 444                 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 445                                                     D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
 446                         d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 447                                                         D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
 448                         d40d->lcla_alloc--;
 449                         if (d40d->lcla_alloc == 0) {
 450                                 ret = 0;
 451                                 break;
 452                         }
 453                 }
 454         }
 455
 456         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 457
 458         return ret;
 459
 460 }
 461
 462 static void d40_desc_remove(struct d40_desc *d40d)
 463 {
 464         list_del(&d40d->node);
 465 }
 466
 467 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 468 {
 469         struct d40_desc *desc = NULL;
 470
 471         if (!list_empty(&d40c->client)) {
 472                 struct d40_desc *d;
 473                 struct d40_desc *_d;
 474
 475                 list_for_each_entry_safe(d, _d, &d40c->client, node)
 476                         if (async_tx_test_ack(&d->txd)) {
 477                                 d40_pool_lli_free(d40c, d);
 478                                 d40_desc_remove(d);
 479                                 desc = d;
 480                                 memset(desc, 0, sizeof(*desc));
 481                                 break;
 482                         }
 483         }
 484
 485         if (!desc)
 486                 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
 487
 488         if (desc)
 489                 INIT_LIST_HEAD(&desc->node);
 490
 491         return desc;
 492 }
 493
 494 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 495 {
 496
 497         d40_pool_lli_free(d40c, d40d);
 498         d40_lcla_free_all(d40c, d40d);
 499         kmem_cache_free(d40c->base->desc_slab, d40d);
 500 }
 501
 502 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 503 {
 504         list_add_tail(&desc->node, &d40c->active);
 505 }
 506
 507 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 508 {
 509         int curr_lcla = -EINVAL, next_lcla;
 510
 511         if (chan_is_physical(d40c)) {
 512                 d40_phy_lli_write(d40c->base->virtbase,
 513                                   d40c->phy_chan->num,
 514                                   d40d->lli_phy.dst,
 515                                   d40d->lli_phy.src);
 516                 d40d->lli_current = d40d->lli_len;
 517         } else {
 518
 519                 if ((d40d->lli_len - d40d->lli_current) > 1)
 520                         curr_lcla = d40_lcla_alloc_one(d40c, d40d);
 521
 522                 d40_log_lli_lcpa_write(d40c->lcpa,
 523                                        &d40d->lli_log.dst[d40d->lli_current],
 524                                        &d40d->lli_log.src[d40d->lli_current],
 525                                        curr_lcla);
 526
 527                 d40d->lli_current++;
 528                 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
 529                         unsigned int lcla_offset = d40c->phy_chan->num * 1024 +
 530                                                    8 * curr_lcla * 2;
 531                         struct d40_lcla_pool *pool = &d40c->base->lcla_pool;
 532                         struct d40_log_lli *lcla = pool->base + lcla_offset;
 533
 534                         if (d40d->lli_current + 1 < d40d->lli_len)
 535                                 next_lcla = d40_lcla_alloc_one(d40c, d40d);
 536                         else
 537                                 next_lcla = -EINVAL;
 538
 539                         d40_log_lli_lcla_write(lcla,
 540                                                &d40d->lli_log.dst[d40d->lli_current],
 541                                                &d40d->lli_log.src[d40d->lli_current],
 542                                                next_lcla);
 543
 544                         dma_sync_single_range_for_device(d40c->base->dev,
 545                                                 pool->dma_addr, lcla_offset,
 546                                                 2 * sizeof(struct d40_log_lli),
 547                                                 DMA_TO_DEVICE);
 548
 549                         curr_lcla = next_lcla;
 550
 551                         if (curr_lcla == -EINVAL) {
 552                                 d40d->lli_current++;
 553                                 break;
 554                         }
 555
 556                 }
 557         }
 558 }
 559
 560 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 561 {
 562         struct d40_desc *d;
 563
 564         if (list_empty(&d40c->active))
 565                 return NULL;
 566
 567         d = list_first_entry(&d40c->active,
 568                              struct d40_desc,
 569                              node);
 570         return d;
 571 }
 572
 573 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 574 {
 575         list_add_tail(&desc->node, &d40c->queue);
 576 }
 577
 578 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 579 {
 580         struct d40_desc *d;
 581
 582         if (list_empty(&d40c->queue))
 583                 return NULL;
 584
 585         d = list_first_entry(&d40c->queue,
 586                              struct d40_desc,
 587                              node);
 588         return d;
 589 }
 590
 591 static int d40_psize_2_burst_size(bool is_log, int psize)
 592 {
 593         if (is_log) {
 594                 if (psize == STEDMA40_PSIZE_LOG_1)
 595                         return 1;
 596         } else {
 597                 if (psize == STEDMA40_PSIZE_PHY_1)
 598                         return 1;
 599         }
 600
 601         return 2 << psize;
 602 }
 603
 604 /*
 605  * The dma only supports transmitting packages up to
 606  * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
 607  * dma elements required to send the entire sg list
 608  */
 609 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
 610 {
 611         int dmalen;
 612         u32 max_w = max(data_width1, data_width2);
 613         u32 min_w = min(data_width1, data_width2);
 614         u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
 615
 616         if (seg_max > STEDMA40_MAX_SEG_SIZE)
 617                 seg_max -= (1 << max_w);
 618
 619         if (!IS_ALIGNED(size, 1 << max_w))
 620                 return -EINVAL;
 621
 622         if (size <= seg_max)
 623                 dmalen = 1;
 624         else {
 625                 dmalen = size / seg_max;
 626                 if (dmalen * seg_max < size)
 627                         dmalen++;
 628         }
 629         return dmalen;
 630 }
 631
 632 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
 633                            u32 data_width1, u32 data_width2)
 634 {
 635         struct scatterlist *sg;
 636         int i;
 637         int len = 0;
 638         int ret;
 639
 640         for_each_sg(sgl, sg, sg_len, i) {
 641                 ret = d40_size_2_dmalen(sg_dma_len(sg),
 642                                         data_width1, data_width2);
 643                 if (ret < 0)
 644                         return ret;
 645                 len += ret;
 646         }
 647         return len;
 648 }
 649
 650 /* Support functions for logical channels */
 651
 652 static int d40_channel_execute_command(struct d40_chan *d40c,
 653                                        enum d40_command command)
 654 {
 655         u32 status;
 656         int i;
 657         void __iomem *active_reg;
 658         int ret = 0;
 659         unsigned long flags;
 660         u32 wmask;
 661
 662         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 663
 664         if (d40c->phy_chan->num % 2 == 0)
 665                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 666         else
 667                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 668
 669         if (command == D40_DMA_SUSPEND_REQ) {
 670                 status = (readl(active_reg) &
 671                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 672                         D40_CHAN_POS(d40c->phy_chan->num);
 673
 674                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 675                         goto done;
 676         }
 677
 678         wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
 679         writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
 680                active_reg);
 681
 682         if (command == D40_DMA_SUSPEND_REQ) {
 683
 684                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 685                         status = (readl(active_reg) &
 686                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 687                                 D40_CHAN_POS(d40c->phy_chan->num);
 688
 689                         cpu_relax();
 690                         /*
 691                          * Reduce the number of bus accesses while
 692                          * waiting for the DMA to suspend.
 693                          */
 694                         udelay(3);
 695
 696                         if (status == D40_DMA_STOP ||
 697                             status == D40_DMA_SUSPENDED)
 698                                 break;
 699                 }
 700
 701                 if (i == D40_SUSPEND_MAX_IT) {
 702                         chan_err(d40c,
 703                                 "unable to suspend the chl %d (log: %d) status %x\n",
 704                                 d40c->phy_chan->num, d40c->log_num,
 705                                 status);
 706                         dump_stack();
 707                         ret = -EBUSY;
 708                 }
 709
 710         }
 711 done:
 712         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 713         return ret;
 714 }
 715
 716 static void d40_term_all(struct d40_chan *d40c)
 717 {
 718         struct d40_desc *d40d;
 719
 720         /* Release active descriptors */
 721         while ((d40d = d40_first_active_get(d40c))) {
 722                 d40_desc_remove(d40d);
 723                 d40_desc_free(d40c, d40d);
 724         }
 725
 726         /* Release queued descriptors waiting for transfer */
 727         while ((d40d = d40_first_queued(d40c))) {
 728                 d40_desc_remove(d40d);
 729                 d40_desc_free(d40c, d40d);
 730         }
 731
 732
 733         d40c->pending_tx = 0;
 734         d40c->busy = false;
 735 }
 736
 737 static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
 738                                    u32 event, int reg)
 739 {
 740         void __iomem *addr = chan_base(d40c) + reg;
 741         int tries;
 742
 743         if (!enable) {
 744                 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 745                        | ~D40_EVENTLINE_MASK(event), addr);
 746                 return;
 747         }
 748
 749         /*
 750          * The hardware sometimes doesn't register the enable when src and dst
 751          * event lines are active on the same logical channel.  Retry to ensure
 752          * it does.  Usually only one retry is sufficient.
 753          */
 754         tries = 100;
 755         while (--tries) {
 756                 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 757                        | ~D40_EVENTLINE_MASK(event), addr);
 758
 759                 if (readl(addr) & D40_EVENTLINE_MASK(event))
 760                         break;
 761         }
 762
 763         if (tries != 99)
 764                 dev_dbg(chan2dev(d40c),
 765                         "[%s] workaround enable S%cLNK (%d tries)\n",
 766                         __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
 767                         100 - tries);
 768
 769         WARN_ON(!tries);
 770 }
 771
 772 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 773 {
 774         unsigned long flags;
 775
 776         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 777
 778         /* Enable event line connected to device (or memcpy) */
 779         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 780             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 781                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 782
 783                 __d40_config_set_event(d40c, do_enable, event,
 784                                        D40_CHAN_REG_SSLNK);
 785         }
 786
 787         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 788                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 789
 790                 __d40_config_set_event(d40c, do_enable, event,
 791                                        D40_CHAN_REG_SDLNK);
 792         }
 793
 794         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 795 }
 796
 797 static u32 d40_chan_has_events(struct d40_chan *d40c)
 798 {
 799         void __iomem *chanbase = chan_base(d40c);
 800         u32 val;
 801
 802         val = readl(chanbase + D40_CHAN_REG_SSLNK);
 803         val |= readl(chanbase + D40_CHAN_REG_SDLNK);
 804
 805         return val;
 806 }
 807
 808 static u32 d40_get_prmo(struct d40_chan *d40c)
 809 {
 810         static const unsigned int phy_map[] = {
 811                 [STEDMA40_PCHAN_BASIC_MODE]
 812                         = D40_DREG_PRMO_PCHAN_BASIC,
 813                 [STEDMA40_PCHAN_MODULO_MODE]
 814                         = D40_DREG_PRMO_PCHAN_MODULO,
 815                 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
 816                         = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
 817         };
 818         static const unsigned int log_map[] = {
 819                 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
 820                         = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
 821                 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
 822                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
 823                 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
 824                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
 825         };
 826
 827         if (chan_is_physical(d40c))
 828                 return phy_map[d40c->dma_cfg.mode_opt];
 829         else
 830                 return log_map[d40c->dma_cfg.mode_opt];
 831 }
 832
 833 static void d40_config_write(struct d40_chan *d40c)
 834 {
 835         u32 addr_base;
 836         u32 var;
 837
 838         /* Odd addresses are even addresses + 4 */
 839         addr_base = (d40c->phy_chan->num % 2) * 4;
 840         /* Setup channel mode to logical or physical */
 841         var = ((u32)(chan_is_logical(d40c)) + 1) <<
 842                 D40_CHAN_POS(d40c->phy_chan->num);
 843         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 844
 845         /* Setup operational mode option register */
 846         var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
 847
 848         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 849
 850         if (chan_is_logical(d40c)) {
 851                 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
 852                            & D40_SREG_ELEM_LOG_LIDX_MASK;
 853                 void __iomem *chanbase = chan_base(d40c);
 854
 855                 /* Set default config for CFG reg */
 856                 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
 857                 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
 858
 859                 /* Set LIDX for lcla */
 860                 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
 861                 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
 862         }
 863 }
 864
 865 static u32 d40_residue(struct d40_chan *d40c)
 866 {
 867         u32 num_elt;
 868
 869         if (chan_is_logical(d40c))
 870                 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
 871                         >> D40_MEM_LCSP2_ECNT_POS;
 872         else {
 873                 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
 874                 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
 875                           >> D40_SREG_ELEM_PHY_ECNT_POS;
 876         }
 877
 878         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
 879 }
 880
 881 static bool d40_tx_is_linked(struct d40_chan *d40c)
 882 {
 883         bool is_link;
 884
 885         if (chan_is_logical(d40c))
 886                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
 887         else
 888                 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
 889                           & D40_SREG_LNK_PHYS_LNK_MASK;
 890
 891         return is_link;
 892 }
 893
 894 static int d40_pause(struct dma_chan *chan)
 895 {
 896         struct d40_chan *d40c =
 897                 container_of(chan, struct d40_chan, chan);
 898         int res = 0;
 899         unsigned long flags;
 900
 901         if (!d40c->busy)
 902                 return 0;
 903
 904         spin_lock_irqsave(&d40c->lock, flags);
 905
 906         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 907         if (res == 0) {
 908                 if (chan_is_logical(d40c)) {
 909                         d40_config_set_event(d40c, false);
 910                         /* Resume the other logical channels if any */
 911                         if (d40_chan_has_events(d40c))
 912                                 res = d40_channel_execute_command(d40c,
 913                                                                   D40_DMA_RUN);
 914                 }
 915         }
 916
 917         spin_unlock_irqrestore(&d40c->lock, flags);
 918         return res;
 919 }
 920
 921 static int d40_resume(struct dma_chan *chan)
 922 {
 923         struct d40_chan *d40c =
 924                 container_of(chan, struct d40_chan, chan);
 925         int res = 0;
 926         unsigned long flags;
 927
 928         if (!d40c->busy)
 929                 return 0;
 930
 931         spin_lock_irqsave(&d40c->lock, flags);
 932
 933         if (d40c->base->rev == 0)
 934                 if (chan_is_logical(d40c)) {
 935                         res = d40_channel_execute_command(d40c,
 936                                                           D40_DMA_SUSPEND_REQ);
 937                         goto no_suspend;
 938                 }
 939
 940         /* If bytes left to transfer or linked tx resume job */
 941         if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
 942
 943                 if (chan_is_logical(d40c))
 944                         d40_config_set_event(d40c, true);
 945
 946                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 947         }
 948
 949 no_suspend:
 950         spin_unlock_irqrestore(&d40c->lock, flags);
 951         return res;
 952 }
 953
 954 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
 955 {
 956         struct d40_chan *d40c = container_of(tx->chan,
 957                                              struct d40_chan,
 958                                              chan);
 959         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
 960         unsigned long flags;
 961
 962         spin_lock_irqsave(&d40c->lock, flags);
 963
 964         d40c->chan.cookie++;
 965
 966         if (d40c->chan.cookie < 0)
 967                 d40c->chan.cookie = 1;
 968
 969         d40d->txd.cookie = d40c->chan.cookie;
 970
 971         d40_desc_queue(d40c, d40d);
 972
 973         spin_unlock_irqrestore(&d40c->lock, flags);
 974
 975         return tx->cookie;
 976 }
 977
 978 static int d40_start(struct d40_chan *d40c)
 979 {
 980         if (d40c->base->rev == 0) {
 981                 int err;
 982
 983                 if (chan_is_logical(d40c)) {
 984                         err = d40_channel_execute_command(d40c,
 985                                                           D40_DMA_SUSPEND_REQ);
 986                         if (err)
 987                                 return err;
 988                 }
 989         }
 990
 991         if (chan_is_logical(d40c))
 992                 d40_config_set_event(d40c, true);
 993
 994         return d40_channel_execute_command(d40c, D40_DMA_RUN);
 995 }
 996
 997 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
 998 {
 999         struct d40_desc *d40d;
1000         int err;
1001
1002         /* Start queued jobs, if any */
1003         d40d = d40_first_queued(d40c);
1004
1005         if (d40d != NULL) {
1006                 d40c->busy = true;
1007
1008                 /* Remove from queue */
1009                 d40_desc_remove(d40d);
1010
1011                 /* Add to active queue */
1012                 d40_desc_submit(d40c, d40d);
1013
1014                 /* Initiate DMA job */
1015                 d40_desc_load(d40c, d40d);
1016
1017                 /* Start dma job */
1018                 err = d40_start(d40c);
1019
1020                 if (err)
1021                         return NULL;
1022         }
1023
1024         return d40d;
1025 }
1026
1027 /* called from interrupt context */
1028 static void dma_tc_handle(struct d40_chan *d40c)
1029 {
1030         struct d40_desc *d40d;
1031
1032         /* Get first active entry from list */
1033         d40d = d40_first_active_get(d40c);
1034
1035         if (d40d == NULL)
1036                 return;
1037
1038         d40_lcla_free_all(d40c, d40d);
1039
1040         if (d40d->lli_current < d40d->lli_len) {
1041                 d40_desc_load(d40c, d40d);
1042                 /* Start dma job */
1043                 (void) d40_start(d40c);
1044                 return;
1045         }
1046
1047         if (d40_queue_start(d40c) == NULL)
1048                 d40c->busy = false;
1049
1050         d40c->pending_tx++;
1051         tasklet_schedule(&d40c->tasklet);
1052
1053 }
1054
1055 static void dma_tasklet(unsigned long data)
1056 {
1057         struct d40_chan *d40c = (struct d40_chan *) data;
1058         struct d40_desc *d40d;
1059         unsigned long flags;
1060         dma_async_tx_callback callback;
1061         void *callback_param;
1062
1063         spin_lock_irqsave(&d40c->lock, flags);
1064
1065         /* Get first active entry from list */
1066         d40d = d40_first_active_get(d40c);
1067
1068         if (d40d == NULL)
1069                 goto err;
1070
1071         d40c->completed = d40d->txd.cookie;
1072
1073         /*
1074          * If terminating a channel pending_tx is set to zero.
1075          * This prevents any finished active jobs to return to the client.
1076          */
1077         if (d40c->pending_tx == 0) {
1078                 spin_unlock_irqrestore(&d40c->lock, flags);
1079                 return;
1080         }
1081
1082         /* Callback to client */
1083         callback = d40d->txd.callback;
1084         callback_param = d40d->txd.callback_param;
1085
1086         if (async_tx_test_ack(&d40d->txd)) {
1087                 d40_pool_lli_free(d40c, d40d);
1088                 d40_desc_remove(d40d);
1089                 d40_desc_free(d40c, d40d);
1090         } else {
1091                 if (!d40d->is_in_client_list) {
1092                         d40_desc_remove(d40d);
1093                         d40_lcla_free_all(d40c, d40d);
1094                         list_add_tail(&d40d->node, &d40c->client);
1095                         d40d->is_in_client_list = true;
1096                 }
1097         }
1098
1099         d40c->pending_tx--;
1100
1101         if (d40c->pending_tx)
1102                 tasklet_schedule(&d40c->tasklet);
1103
1104         spin_unlock_irqrestore(&d40c->lock, flags);
1105
1106         if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1107                 callback(callback_param);
1108
1109         return;
1110
1111  err:
1112         /* Rescue manouver if receiving double interrupts */
1113         if (d40c->pending_tx > 0)
1114                 d40c->pending_tx--;
1115         spin_unlock_irqrestore(&d40c->lock, flags);
1116 }
1117
1118 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1119 {
1120         static const struct d40_interrupt_lookup il[] = {
1121                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
1122                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1123                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1124                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1125                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
1126                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
1127                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
1128                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
1129                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
1130                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
1131         };
1132
1133         int i;
1134         u32 regs[ARRAY_SIZE(il)];
1135         u32 idx;
1136         u32 row;
1137         long chan = -1;
1138         struct d40_chan *d40c;
1139         unsigned long flags;
1140         struct d40_base *base = data;
1141
1142         spin_lock_irqsave(&base->interrupt_lock, flags);
1143
1144         /* Read interrupt status of both logical and physical channels */
1145         for (i = 0; i < ARRAY_SIZE(il); i++)
1146                 regs[i] = readl(base->virtbase + il[i].src);
1147
1148         for (;;) {
1149
1150                 chan = find_next_bit((unsigned long *)regs,
1151                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1152
1153                 /* No more set bits found? */
1154                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1155                         break;
1156
1157                 row = chan / BITS_PER_LONG;
1158                 idx = chan & (BITS_PER_LONG - 1);
1159
1160                 /* ACK interrupt */
1161                 writel(1 << idx, base->virtbase + il[row].clr);
1162
1163                 if (il[row].offset == D40_PHY_CHAN)
1164                         d40c = base->lookup_phy_chans[idx];
1165                 else
1166                         d40c = base->lookup_log_chans[il[row].offset + idx];
1167                 spin_lock(&d40c->lock);
1168
1169                 if (!il[row].is_error)
1170                         dma_tc_handle(d40c);
1171                 else
1172                         d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1173                                 chan, il[row].offset, idx);
1174
1175                 spin_unlock(&d40c->lock);
1176         }
1177
1178         spin_unlock_irqrestore(&base->interrupt_lock, flags);
1179
1180         return IRQ_HANDLED;
1181 }
1182
1183 static int d40_validate_conf(struct d40_chan *d40c,
1184                              struct stedma40_chan_cfg *conf)
1185 {
1186         int res = 0;
1187         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1188         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1189         bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1190
1191         if (!conf->dir) {
1192                 chan_err(d40c, "Invalid direction.\n");
1193                 res = -EINVAL;
1194         }
1195
1196         if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1197             d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1198             d40c->runtime_addr == 0) {
1199
1200                 chan_err(d40c, "Invalid TX channel address (%d)\n",
1201                          conf->dst_dev_type);
1202                 res = -EINVAL;
1203         }
1204
1205         if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1206             d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1207             d40c->runtime_addr == 0) {
1208                 chan_err(d40c, "Invalid RX channel address (%d)\n",
1209                         conf->src_dev_type);
1210                 res = -EINVAL;
1211         }
1212
1213         if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1214             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1215                 chan_err(d40c, "Invalid dst\n");
1216                 res = -EINVAL;
1217         }
1218
1219         if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1220             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1221                 chan_err(d40c, "Invalid src\n");
1222                 res = -EINVAL;
1223         }
1224
1225         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1226             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1227                 chan_err(d40c, "No event line\n");
1228                 res = -EINVAL;
1229         }
1230
1231         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1232             (src_event_group != dst_event_group)) {
1233                 chan_err(d40c, "Invalid event group\n");
1234                 res = -EINVAL;
1235         }
1236
1237         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1238                 /*
1239                  * DMAC HW supports it. Will be added to this driver,
1240                  * in case any dma client requires it.
1241                  */
1242                 chan_err(d40c, "periph to periph not supported\n");
1243                 res = -EINVAL;
1244         }
1245
1246         if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1247             (1 << conf->src_info.data_width) !=
1248             d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1249             (1 << conf->dst_info.data_width)) {
1250                 /*
1251                  * The DMAC hardware only supports
1252                  * src (burst x width) == dst (burst x width)
1253                  */
1254
1255                 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1256                 res = -EINVAL;
1257         }
1258
1259         return res;
1260 }
1261
1262 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1263                                int log_event_line, bool is_log)
1264 {
1265         unsigned long flags;
1266         spin_lock_irqsave(&phy->lock, flags);
1267         if (!is_log) {
1268                 /* Physical interrupts are masked per physical full channel */
1269                 if (phy->allocated_src == D40_ALLOC_FREE &&
1270                     phy->allocated_dst == D40_ALLOC_FREE) {
1271                         phy->allocated_dst = D40_ALLOC_PHY;
1272                         phy->allocated_src = D40_ALLOC_PHY;
1273                         goto found;
1274                 } else
1275                         goto not_found;
1276         }
1277
1278         /* Logical channel */
1279         if (is_src) {
1280                 if (phy->allocated_src == D40_ALLOC_PHY)
1281                         goto not_found;
1282
1283                 if (phy->allocated_src == D40_ALLOC_FREE)
1284                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1285
1286                 if (!(phy->allocated_src & (1 << log_event_line))) {
1287                         phy->allocated_src |= 1 << log_event_line;
1288                         goto found;
1289                 } else
1290                         goto not_found;
1291         } else {
1292                 if (phy->allocated_dst == D40_ALLOC_PHY)
1293                         goto not_found;
1294
1295                 if (phy->allocated_dst == D40_ALLOC_FREE)
1296                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1297
1298                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1299                         phy->allocated_dst |= 1 << log_event_line;
1300                         goto found;
1301                 } else
1302                         goto not_found;
1303         }
1304
1305 not_found:
1306         spin_unlock_irqrestore(&phy->lock, flags);
1307         return false;
1308 found:
1309         spin_unlock_irqrestore(&phy->lock, flags);
1310         return true;
1311 }
1312
1313 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1314                                int log_event_line)
1315 {
1316         unsigned long flags;
1317         bool is_free = false;
1318
1319         spin_lock_irqsave(&phy->lock, flags);
1320         if (!log_event_line) {
1321                 phy->allocated_dst = D40_ALLOC_FREE;
1322                 phy->allocated_src = D40_ALLOC_FREE;
1323                 is_free = true;
1324                 goto out;
1325         }
1326
1327         /* Logical channel */
1328         if (is_src) {
1329                 phy->allocated_src &= ~(1 << log_event_line);
1330                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1331                         phy->allocated_src = D40_ALLOC_FREE;
1332         } else {
1333                 phy->allocated_dst &= ~(1 << log_event_line);
1334                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1335                         phy->allocated_dst = D40_ALLOC_FREE;
1336         }
1337
1338         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1339                    D40_ALLOC_FREE);
1340
1341 out:
1342         spin_unlock_irqrestore(&phy->lock, flags);
1343
1344         return is_free;
1345 }
1346
1347 static int d40_allocate_channel(struct d40_chan *d40c)
1348 {
1349         int dev_type;
1350         int event_group;
1351         int event_line;
1352         struct d40_phy_res *phys;
1353         int i;
1354         int j;
1355         int log_num;
1356         bool is_src;
1357         bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1358
1359         phys = d40c->base->phy_res;
1360
1361         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1362                 dev_type = d40c->dma_cfg.src_dev_type;
1363                 log_num = 2 * dev_type;
1364                 is_src = true;
1365         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1366                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1367                 /* dst event lines are used for logical memcpy */
1368                 dev_type = d40c->dma_cfg.dst_dev_type;
1369                 log_num = 2 * dev_type + 1;
1370                 is_src = false;
1371         } else
1372                 return -EINVAL;
1373
1374         event_group = D40_TYPE_TO_GROUP(dev_type);
1375         event_line = D40_TYPE_TO_EVENT(dev_type);
1376
1377         if (!is_log) {
1378                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1379                         /* Find physical half channel */
1380                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1381
1382                                 if (d40_alloc_mask_set(&phys[i], is_src,
1383                                                        0, is_log))
1384                                         goto found_phy;
1385                         }
1386                 } else
1387                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1388                                 int phy_num = j  + event_group * 2;
1389                                 for (i = phy_num; i < phy_num + 2; i++) {
1390                                         if (d40_alloc_mask_set(&phys[i],
1391                                                                is_src,
1392                                                                0,
1393                                                                is_log))
1394                                                 goto found_phy;
1395                                 }
1396                         }
1397                 return -EINVAL;
1398 found_phy:
1399                 d40c->phy_chan = &phys[i];
1400                 d40c->log_num = D40_PHY_CHAN;
1401                 goto out;
1402         }
1403         if (dev_type == -1)
1404                 return -EINVAL;
1405
1406         /* Find logical channel */
1407         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1408                 int phy_num = j + event_group * 2;
1409                 /*
1410                  * Spread logical channels across all available physical rather
1411                  * than pack every logical channel at the first available phy
1412                  * channels.
1413                  */
1414                 if (is_src) {
1415                         for (i = phy_num; i < phy_num + 2; i++) {
1416                                 if (d40_alloc_mask_set(&phys[i], is_src,
1417                                                        event_line, is_log))
1418                                         goto found_log;
1419                         }
1420                 } else {
1421                         for (i = phy_num + 1; i >= phy_num; i--) {
1422                                 if (d40_alloc_mask_set(&phys[i], is_src,
1423                                                        event_line, is_log))
1424                                         goto found_log;
1425                         }
1426                 }
1427         }
1428         return -EINVAL;
1429
1430 found_log:
1431         d40c->phy_chan = &phys[i];
1432         d40c->log_num = log_num;
1433 out:
1434
1435         if (is_log)
1436                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1437         else
1438                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1439
1440         return 0;
1441
1442 }
1443
1444 static int d40_config_memcpy(struct d40_chan *d40c)
1445 {
1446         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1447
1448         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1449                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1450                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1451                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1452                         memcpy[d40c->chan.chan_id];
1453
1454         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1455                    dma_has_cap(DMA_SLAVE, cap)) {
1456                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1457         } else {
1458                 chan_err(d40c, "No memcpy\n");
1459                 return -EINVAL;
1460         }
1461
1462         return 0;
1463 }
1464
1465
1466 static int d40_free_dma(struct d40_chan *d40c)
1467 {
1468
1469         int res = 0;
1470         u32 event;
1471         struct d40_phy_res *phy = d40c->phy_chan;
1472         bool is_src;
1473         struct d40_desc *d;
1474         struct d40_desc *_d;
1475
1476
1477         /* Terminate all queued and active transfers */
1478         d40_term_all(d40c);
1479
1480         /* Release client owned descriptors */
1481         if (!list_empty(&d40c->client))
1482                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1483                         d40_pool_lli_free(d40c, d);
1484                         d40_desc_remove(d);
1485                         d40_desc_free(d40c, d);
1486                 }
1487
1488         if (phy == NULL) {
1489                 chan_err(d40c, "phy == null\n");
1490                 return -EINVAL;
1491         }
1492
1493         if (phy->allocated_src == D40_ALLOC_FREE &&
1494             phy->allocated_dst == D40_ALLOC_FREE) {
1495                 chan_err(d40c, "channel already free\n");
1496                 return -EINVAL;
1497         }
1498
1499         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1500             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1501                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1502                 is_src = false;
1503         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1504                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1505                 is_src = true;
1506         } else {
1507                 chan_err(d40c, "Unknown direction\n");
1508                 return -EINVAL;
1509         }
1510
1511         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1512         if (res) {
1513                 chan_err(d40c, "suspend failed\n");
1514                 return res;
1515         }
1516
1517         if (chan_is_logical(d40c)) {
1518                 /* Release logical channel, deactivate the event line */
1519
1520                 d40_config_set_event(d40c, false);
1521                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1522
1523                 /*
1524                  * Check if there are more logical allocation
1525                  * on this phy channel.
1526                  */
1527                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1528                         /* Resume the other logical channels if any */
1529                         if (d40_chan_has_events(d40c)) {
1530                                 res = d40_channel_execute_command(d40c,
1531                                                                   D40_DMA_RUN);
1532                                 if (res) {
1533                                         chan_err(d40c,
1534                                                 "Executing RUN command\n");
1535                                         return res;
1536                                 }
1537                         }
1538                         return 0;
1539                 }
1540         } else {
1541                 (void) d40_alloc_mask_free(phy, is_src, 0);
1542         }
1543
1544         /* Release physical channel */
1545         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1546         if (res) {
1547                 chan_err(d40c, "Failed to stop channel\n");
1548                 return res;
1549         }
1550         d40c->phy_chan = NULL;
1551         d40c->configured = false;
1552         d40c->base->lookup_phy_chans[phy->num] = NULL;
1553
1554         return 0;
1555 }
1556
1557 static bool d40_is_paused(struct d40_chan *d40c)
1558 {
1559         void __iomem *chanbase = chan_base(d40c);
1560         bool is_paused = false;
1561         unsigned long flags;
1562         void __iomem *active_reg;
1563         u32 status;
1564         u32 event;
1565
1566         spin_lock_irqsave(&d40c->lock, flags);
1567
1568         if (chan_is_physical(d40c)) {
1569                 if (d40c->phy_chan->num % 2 == 0)
1570                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1571                 else
1572                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1573
1574                 status = (readl(active_reg) &
1575                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1576                         D40_CHAN_POS(d40c->phy_chan->num);
1577                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1578                         is_paused = true;
1579
1580                 goto _exit;
1581         }
1582
1583         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1584             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1585                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1586                 status = readl(chanbase + D40_CHAN_REG_SDLNK);
1587         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1588                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1589                 status = readl(chanbase + D40_CHAN_REG_SSLNK);
1590         } else {
1591                 chan_err(d40c, "Unknown direction\n");
1592                 goto _exit;
1593         }
1594
1595         status = (status & D40_EVENTLINE_MASK(event)) >>
1596                 D40_EVENTLINE_POS(event);
1597
1598         if (status != D40_DMA_RUN)
1599                 is_paused = true;
1600 _exit:
1601         spin_unlock_irqrestore(&d40c->lock, flags);
1602         return is_paused;
1603
1604 }
1605
1606
1607 static u32 stedma40_residue(struct dma_chan *chan)
1608 {
1609         struct d40_chan *d40c =
1610                 container_of(chan, struct d40_chan, chan);
1611         u32 bytes_left;
1612         unsigned long flags;
1613
1614         spin_lock_irqsave(&d40c->lock, flags);
1615         bytes_left = d40_residue(d40c);
1616         spin_unlock_irqrestore(&d40c->lock, flags);
1617
1618         return bytes_left;
1619 }
1620
1621 static struct d40_desc *
1622 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1623               unsigned int sg_len, unsigned long dma_flags)
1624 {
1625         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1626         struct d40_desc *desc;
1627         int ret;
1628
1629         desc = d40_desc_get(chan);
1630         if (!desc)
1631                 return NULL;
1632
1633         desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1634                                         cfg->dst_info.data_width);
1635         if (desc->lli_len < 0) {
1636                 chan_err(chan, "Unaligned size\n");
1637                 goto err;
1638         }
1639
1640         ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
1641         if (ret < 0) {
1642                 chan_err(chan, "Could not allocate lli\n");
1643                 goto err;
1644         }
1645
1646
1647         desc->lli_current = 0;
1648         desc->txd.flags = dma_flags;
1649         desc->txd.tx_submit = d40_tx_submit;
1650
1651         dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1652
1653         return desc;
1654
1655 err:
1656         d40_desc_free(chan, desc);
1657         return NULL;
1658 }
1659
1660 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1661                                                    struct scatterlist *sgl_dst,
1662                                                    struct scatterlist *sgl_src,
1663                                                    unsigned int sgl_len,
1664                                                    unsigned long dma_flags)
1665 {
1666         int res;
1667         struct d40_desc *d40d;
1668         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1669                                              chan);
1670         unsigned long flags;
1671
1672         if (d40c->phy_chan == NULL) {
1673                 chan_err(d40c, "Unallocated channel.\n");
1674                 return ERR_PTR(-EINVAL);
1675         }
1676
1677         spin_lock_irqsave(&d40c->lock, flags);
1678
1679         d40d = d40_prep_desc(d40c, sgl_dst, sgl_len, dma_flags);
1680         if (!d40d)
1681                 goto err;
1682
1683         if (chan_is_logical(d40c)) {
1684                 (void) d40_log_sg_to_lli(sgl_src,
1685                                          sgl_len,
1686                                          d40d->lli_log.src,
1687                                          d40c->log_def.lcsp1,
1688                                          d40c->dma_cfg.src_info.data_width,
1689                                          d40c->dma_cfg.dst_info.data_width);
1690
1691                 (void) d40_log_sg_to_lli(sgl_dst,
1692                                          sgl_len,
1693                                          d40d->lli_log.dst,
1694                                          d40c->log_def.lcsp3,
1695                                          d40c->dma_cfg.dst_info.data_width,
1696                                          d40c->dma_cfg.src_info.data_width);
1697         } else {
1698                 res = d40_phy_sg_to_lli(sgl_src,
1699                                         sgl_len,
1700                                         0,
1701                                         d40d->lli_phy.src,
1702                                         virt_to_phys(d40d->lli_phy.src),
1703                                         d40c->src_def_cfg,
1704                                         d40c->dma_cfg.src_info.data_width,
1705                                         d40c->dma_cfg.dst_info.data_width,
1706                                         d40c->dma_cfg.src_info.psize);
1707
1708                 if (res < 0)
1709                         goto err;
1710
1711                 res = d40_phy_sg_to_lli(sgl_dst,
1712                                         sgl_len,
1713                                         0,
1714                                         d40d->lli_phy.dst,
1715                                         virt_to_phys(d40d->lli_phy.dst),
1716                                         d40c->dst_def_cfg,
1717                                         d40c->dma_cfg.dst_info.data_width,
1718                                         d40c->dma_cfg.src_info.data_width,
1719                                         d40c->dma_cfg.dst_info.psize);
1720
1721                 if (res < 0)
1722                         goto err;
1723
1724                 dma_sync_single_for_device(d40c->base->dev,
1725                                            d40d->lli_pool.dma_addr,
1726                                            d40d->lli_pool.size, DMA_TO_DEVICE);
1727         }
1728
1729         spin_unlock_irqrestore(&d40c->lock, flags);
1730
1731         return &d40d->txd;
1732 err:
1733         if (d40d)
1734                 d40_desc_free(d40c, d40d);
1735         spin_unlock_irqrestore(&d40c->lock, flags);
1736         return NULL;
1737 }
1738 EXPORT_SYMBOL(stedma40_memcpy_sg);
1739
1740 bool stedma40_filter(struct dma_chan *chan, void *data)
1741 {
1742         struct stedma40_chan_cfg *info = data;
1743         struct d40_chan *d40c =
1744                 container_of(chan, struct d40_chan, chan);
1745         int err;
1746
1747         if (data) {
1748                 err = d40_validate_conf(d40c, info);
1749                 if (!err)
1750                         d40c->dma_cfg = *info;
1751         } else
1752                 err = d40_config_memcpy(d40c);
1753
1754         if (!err)
1755                 d40c->configured = true;
1756
1757         return err == 0;
1758 }
1759 EXPORT_SYMBOL(stedma40_filter);
1760
1761 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1762 {
1763         bool realtime = d40c->dma_cfg.realtime;
1764         bool highprio = d40c->dma_cfg.high_priority;
1765         u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1766         u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1767         u32 event = D40_TYPE_TO_EVENT(dev_type);
1768         u32 group = D40_TYPE_TO_GROUP(dev_type);
1769         u32 bit = 1 << event;
1770
1771         /* Destination event lines are stored in the upper halfword */
1772         if (!src)
1773                 bit <<= 16;
1774
1775         writel(bit, d40c->base->virtbase + prioreg + group * 4);
1776         writel(bit, d40c->base->virtbase + rtreg + group * 4);
1777 }
1778
1779 static void d40_set_prio_realtime(struct d40_chan *d40c)
1780 {
1781         if (d40c->base->rev < 3)
1782                 return;
1783
1784         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
1785             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1786                 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1787
1788         if ((d40c->dma_cfg.dir ==  STEDMA40_MEM_TO_PERIPH) ||
1789             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1790                 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1791 }
1792
1793 /* DMA ENGINE functions */
1794 static int d40_alloc_chan_resources(struct dma_chan *chan)
1795 {
1796         int err;
1797         unsigned long flags;
1798         struct d40_chan *d40c =
1799                 container_of(chan, struct d40_chan, chan);
1800         bool is_free_phy;
1801         spin_lock_irqsave(&d40c->lock, flags);
1802
1803         d40c->completed = chan->cookie = 1;
1804
1805         /* If no dma configuration is set use default configuration (memcpy) */
1806         if (!d40c->configured) {
1807                 err = d40_config_memcpy(d40c);
1808                 if (err) {
1809                         chan_err(d40c, "Failed to configure memcpy channel\n");
1810                         goto fail;
1811                 }
1812         }
1813         is_free_phy = (d40c->phy_chan == NULL);
1814
1815         err = d40_allocate_channel(d40c);
1816         if (err) {
1817                 chan_err(d40c, "Failed to allocate channel\n");
1818                 goto fail;
1819         }
1820
1821         /* Fill in basic CFG register values */
1822         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1823                     &d40c->dst_def_cfg, chan_is_logical(d40c));
1824
1825         d40_set_prio_realtime(d40c);
1826
1827         if (chan_is_logical(d40c)) {
1828                 d40_log_cfg(&d40c->dma_cfg,
1829                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1830
1831                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1832                         d40c->lcpa = d40c->base->lcpa_base +
1833                           d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1834                 else
1835                         d40c->lcpa = d40c->base->lcpa_base +
1836                           d40c->dma_cfg.dst_dev_type *
1837                           D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1838         }
1839
1840         /*
1841          * Only write channel configuration to the DMA if the physical
1842          * resource is free. In case of multiple logical channels
1843          * on the same physical resource, only the first write is necessary.
1844          */
1845         if (is_free_phy)
1846                 d40_config_write(d40c);
1847 fail:
1848         spin_unlock_irqrestore(&d40c->lock, flags);
1849         return err;
1850 }
1851
1852 static void d40_free_chan_resources(struct dma_chan *chan)
1853 {
1854         struct d40_chan *d40c =
1855                 container_of(chan, struct d40_chan, chan);
1856         int err;
1857         unsigned long flags;
1858
1859         if (d40c->phy_chan == NULL) {
1860                 chan_err(d40c, "Cannot free unallocated channel\n");
1861                 return;
1862         }
1863
1864
1865         spin_lock_irqsave(&d40c->lock, flags);
1866
1867         err = d40_free_dma(d40c);
1868
1869         if (err)
1870                 chan_err(d40c, "Failed to free channel\n");
1871         spin_unlock_irqrestore(&d40c->lock, flags);
1872 }
1873
1874 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1875                                                        dma_addr_t dst,
1876                                                        dma_addr_t src,
1877                                                        size_t size,
1878                                                        unsigned long dma_flags)
1879 {
1880         struct scatterlist dst_sg;
1881         struct scatterlist src_sg;
1882
1883         sg_init_table(&dst_sg, 1);
1884         sg_init_table(&src_sg, 1);
1885
1886         sg_dma_address(&dst_sg) = dst;
1887         sg_dma_address(&src_sg) = src;
1888
1889         sg_dma_len(&dst_sg) = size;
1890         sg_dma_len(&src_sg) = size;
1891
1892         return stedma40_memcpy_sg(chan, &dst_sg, &src_sg, 1, dma_flags);
1893 }
1894
1895 static struct dma_async_tx_descriptor *
1896 d40_prep_sg(struct dma_chan *chan,
1897             struct scatterlist *dst_sg, unsigned int dst_nents,
1898             struct scatterlist *src_sg, unsigned int src_nents,
1899             unsigned long dma_flags)
1900 {
1901         if (dst_nents != src_nents)
1902                 return NULL;
1903
1904         return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
1905 }
1906
1907 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1908                                  struct d40_chan *d40c,
1909                                  struct scatterlist *sgl,
1910                                  unsigned int sg_len,
1911                                  enum dma_data_direction direction,
1912                                  dma_addr_t dev_addr)
1913 {
1914         int total_size;
1915
1916         total_size = d40_log_sg_to_dev(sgl, sg_len,
1917                                        &d40d->lli_log,
1918                                        &d40c->log_def,
1919                                        d40c->dma_cfg.src_info.data_width,
1920                                        d40c->dma_cfg.dst_info.data_width,
1921                                        direction,
1922                                        dev_addr);
1923
1924         if (total_size < 0)
1925                 return -EINVAL;
1926
1927         return 0;
1928 }
1929
1930 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1931                                  struct d40_chan *d40c,
1932                                  struct scatterlist *sgl,
1933                                  unsigned int sgl_len,
1934                                  enum dma_data_direction direction,
1935                                  dma_addr_t dev_addr)
1936 {
1937         dma_addr_t src_dev_addr = direction == DMA_FROM_DEVICE ? dev_addr : 0;
1938         dma_addr_t dst_dev_addr = direction == DMA_TO_DEVICE ? dev_addr : 0;
1939         int res;
1940
1941         res = d40_phy_sg_to_lli(sgl,
1942                                 sgl_len,
1943                                 src_dev_addr,
1944                                 d40d->lli_phy.src,
1945                                 virt_to_phys(d40d->lli_phy.src),
1946                                 d40c->src_def_cfg,
1947                                 d40c->dma_cfg.src_info.data_width,
1948                                 d40c->dma_cfg.dst_info.data_width,
1949                                 d40c->dma_cfg.src_info.psize);
1950         if (res < 0)
1951                 return res;
1952
1953         res = d40_phy_sg_to_lli(sgl,
1954                                 sgl_len,
1955                                 dst_dev_addr,
1956                                 d40d->lli_phy.dst,
1957                                 virt_to_phys(d40d->lli_phy.dst),
1958                                 d40c->dst_def_cfg,
1959                                 d40c->dma_cfg.dst_info.data_width,
1960                                 d40c->dma_cfg.src_info.data_width,
1961                                 d40c->dma_cfg.dst_info.psize);
1962         if (res < 0)
1963                 return res;
1964
1965         dma_sync_single_for_device(d40c->base->dev, d40d->lli_pool.dma_addr,
1966                                    d40d->lli_pool.size, DMA_TO_DEVICE);
1967         return 0;
1968 }
1969
1970 static dma_addr_t
1971 d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
1972 {
1973         struct stedma40_platform_data *plat = chan->base->plat_data;
1974         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1975         dma_addr_t addr;
1976
1977         if (chan->runtime_addr)
1978                 return chan->runtime_addr;
1979
1980         if (direction == DMA_FROM_DEVICE)
1981                 addr = plat->dev_rx[cfg->src_dev_type];
1982         else if (direction == DMA_TO_DEVICE)
1983                 addr = plat->dev_tx[cfg->dst_dev_type];
1984
1985         return addr;
1986 }
1987
1988 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
1989                                                          struct scatterlist *sgl,
1990                                                          unsigned int sg_len,
1991                                                          enum dma_data_direction direction,
1992                                                          unsigned long dma_flags)
1993 {
1994         struct d40_desc *d40d;
1995         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1996                                              chan);
1997         dma_addr_t dev_addr;
1998         unsigned long flags;
1999         int err;
2000
2001         if (d40c->phy_chan == NULL) {
2002                 chan_err(d40c, "Cannot prepare unallocated channel\n");
2003                 return ERR_PTR(-EINVAL);
2004         }
2005
2006         if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
2007                 return NULL;
2008
2009         spin_lock_irqsave(&d40c->lock, flags);
2010
2011         d40d = d40_prep_desc(d40c, sgl, sg_len, dma_flags);
2012         if (d40d == NULL)
2013                 goto err;
2014
2015         dev_addr = d40_get_dev_addr(d40c, direction);
2016
2017         if (chan_is_logical(d40c))
2018                 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2019                                             direction, dev_addr);
2020         else
2021                 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2022                                             direction, dev_addr);
2023         if (err) {
2024                 chan_err(d40c, "Failed to prepare %s slave sg job: %d\n",
2025                         chan_is_logical(d40c) ? "log" : "phy", err);
2026                 goto err;
2027         }
2028
2029         spin_unlock_irqrestore(&d40c->lock, flags);
2030         return &d40d->txd;
2031
2032 err:
2033         if (d40d)
2034                 d40_desc_free(d40c, d40d);
2035         spin_unlock_irqrestore(&d40c->lock, flags);
2036         return NULL;
2037 }
2038
2039 static enum dma_status d40_tx_status(struct dma_chan *chan,
2040                                      dma_cookie_t cookie,
2041                                      struct dma_tx_state *txstate)
2042 {
2043         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2044         dma_cookie_t last_used;
2045         dma_cookie_t last_complete;
2046         int ret;
2047
2048         if (d40c->phy_chan == NULL) {
2049                 chan_err(d40c, "Cannot read status of unallocated channel\n");
2050                 return -EINVAL;
2051         }
2052
2053         last_complete = d40c->completed;
2054         last_used = chan->cookie;
2055
2056         if (d40_is_paused(d40c))
2057                 ret = DMA_PAUSED;
2058         else
2059                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2060
2061         dma_set_tx_state(txstate, last_complete, last_used,
2062                          stedma40_residue(chan));
2063
2064         return ret;
2065 }
2066
2067 static void d40_issue_pending(struct dma_chan *chan)
2068 {
2069         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2070         unsigned long flags;
2071
2072         if (d40c->phy_chan == NULL) {
2073                 chan_err(d40c, "Channel is not allocated!\n");
2074                 return;
2075         }
2076
2077         spin_lock_irqsave(&d40c->lock, flags);
2078
2079         /* Busy means that pending jobs are already being processed */
2080         if (!d40c->busy)
2081                 (void) d40_queue_start(d40c);
2082
2083         spin_unlock_irqrestore(&d40c->lock, flags);
2084 }
2085
2086 /* Runtime reconfiguration extension */
2087 static void d40_set_runtime_config(struct dma_chan *chan,
2088                                struct dma_slave_config *config)
2089 {
2090         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2091         struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2092         enum dma_slave_buswidth config_addr_width;
2093         dma_addr_t config_addr;
2094         u32 config_maxburst;
2095         enum stedma40_periph_data_width addr_width;
2096         int psize;
2097
2098         if (config->direction == DMA_FROM_DEVICE) {
2099                 dma_addr_t dev_addr_rx =
2100                         d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2101
2102                 config_addr = config->src_addr;
2103                 if (dev_addr_rx)
2104                         dev_dbg(d40c->base->dev,
2105                                 "channel has a pre-wired RX address %08x "
2106                                 "overriding with %08x\n",
2107                                 dev_addr_rx, config_addr);
2108                 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2109                         dev_dbg(d40c->base->dev,
2110                                 "channel was not configured for peripheral "
2111                                 "to memory transfer (%d) overriding\n",
2112                                 cfg->dir);
2113                 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2114
2115                 config_addr_width = config->src_addr_width;
2116                 config_maxburst = config->src_maxburst;
2117
2118         } else if (config->direction == DMA_TO_DEVICE) {
2119                 dma_addr_t dev_addr_tx =
2120                         d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2121
2122                 config_addr = config->dst_addr;
2123                 if (dev_addr_tx)
2124                         dev_dbg(d40c->base->dev,
2125                                 "channel has a pre-wired TX address %08x "
2126                                 "overriding with %08x\n",
2127                                 dev_addr_tx, config_addr);
2128                 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2129                         dev_dbg(d40c->base->dev,
2130                                 "channel was not configured for memory "
2131                                 "to peripheral transfer (%d) overriding\n",
2132                                 cfg->dir);
2133                 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2134
2135                 config_addr_width = config->dst_addr_width;
2136                 config_maxburst = config->dst_maxburst;
2137
2138         } else {
2139                 dev_err(d40c->base->dev,
2140                         "unrecognized channel direction %d\n",
2141                         config->direction);
2142                 return;
2143         }
2144
2145         switch (config_addr_width) {
2146         case DMA_SLAVE_BUSWIDTH_1_BYTE:
2147                 addr_width = STEDMA40_BYTE_WIDTH;
2148                 break;
2149         case DMA_SLAVE_BUSWIDTH_2_BYTES:
2150                 addr_width = STEDMA40_HALFWORD_WIDTH;
2151                 break;
2152         case DMA_SLAVE_BUSWIDTH_4_BYTES:
2153                 addr_width = STEDMA40_WORD_WIDTH;
2154                 break;
2155         case DMA_SLAVE_BUSWIDTH_8_BYTES:
2156                 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2157                 break;
2158         default:
2159                 dev_err(d40c->base->dev,
2160                         "illegal peripheral address width "
2161                         "requested (%d)\n",
2162                         config->src_addr_width);
2163                 return;
2164         }
2165
2166         if (chan_is_logical(d40c)) {
2167                 if (config_maxburst >= 16)
2168                         psize = STEDMA40_PSIZE_LOG_16;
2169                 else if (config_maxburst >= 8)
2170                         psize = STEDMA40_PSIZE_LOG_8;
2171                 else if (config_maxburst >= 4)
2172                         psize = STEDMA40_PSIZE_LOG_4;
2173                 else
2174                         psize = STEDMA40_PSIZE_LOG_1;
2175         } else {
2176                 if (config_maxburst >= 16)
2177                         psize = STEDMA40_PSIZE_PHY_16;
2178                 else if (config_maxburst >= 8)
2179                         psize = STEDMA40_PSIZE_PHY_8;
2180                 else if (config_maxburst >= 4)
2181                         psize = STEDMA40_PSIZE_PHY_4;
2182                 else if (config_maxburst >= 2)
2183                         psize = STEDMA40_PSIZE_PHY_2;
2184                 else
2185                         psize = STEDMA40_PSIZE_PHY_1;
2186         }
2187
2188         /* Set up all the endpoint configs */
2189         cfg->src_info.data_width = addr_width;
2190         cfg->src_info.psize = psize;
2191         cfg->src_info.big_endian = false;
2192         cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2193         cfg->dst_info.data_width = addr_width;
2194         cfg->dst_info.psize = psize;
2195         cfg->dst_info.big_endian = false;
2196         cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2197
2198         /* Fill in register values */
2199         if (chan_is_logical(d40c))
2200                 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2201         else
2202                 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2203                             &d40c->dst_def_cfg, false);
2204
2205         /* These settings will take precedence later */
2206         d40c->runtime_addr = config_addr;
2207         d40c->runtime_direction = config->direction;
2208         dev_dbg(d40c->base->dev,
2209                 "configured channel %s for %s, data width %d, "
2210                 "maxburst %d bytes, LE, no flow control\n",
2211                 dma_chan_name(chan),
2212                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2213                 config_addr_width,
2214                 config_maxburst);
2215 }
2216
2217 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2218                        unsigned long arg)
2219 {
2220         unsigned long flags;
2221         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2222
2223         if (d40c->phy_chan == NULL) {
2224                 chan_err(d40c, "Channel is not allocated!\n");
2225                 return -EINVAL;
2226         }
2227
2228         switch (cmd) {
2229         case DMA_TERMINATE_ALL:
2230                 spin_lock_irqsave(&d40c->lock, flags);
2231                 d40_term_all(d40c);
2232                 spin_unlock_irqrestore(&d40c->lock, flags);
2233                 return 0;
2234         case DMA_PAUSE:
2235                 return d40_pause(chan);
2236         case DMA_RESUME:
2237                 return d40_resume(chan);
2238         case DMA_SLAVE_CONFIG:
2239                 d40_set_runtime_config(chan,
2240                         (struct dma_slave_config *) arg);
2241                 return 0;
2242         default:
2243                 break;
2244         }
2245
2246         /* Other commands are unimplemented */
2247         return -ENXIO;
2248 }
2249
2250 /* Initialization functions */
2251
2252 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2253                                  struct d40_chan *chans, int offset,
2254                                  int num_chans)
2255 {
2256         int i = 0;
2257         struct d40_chan *d40c;
2258
2259         INIT_LIST_HEAD(&dma->channels);
2260
2261         for (i = offset; i < offset + num_chans; i++) {
2262                 d40c = &chans[i];
2263                 d40c->base = base;
2264                 d40c->chan.device = dma;
2265
2266                 spin_lock_init(&d40c->lock);
2267
2268                 d40c->log_num = D40_PHY_CHAN;
2269
2270                 INIT_LIST_HEAD(&d40c->active);
2271                 INIT_LIST_HEAD(&d40c->queue);
2272                 INIT_LIST_HEAD(&d40c->client);
2273
2274                 tasklet_init(&d40c->tasklet, dma_tasklet,
2275                              (unsigned long) d40c);
2276
2277                 list_add_tail(&d40c->chan.device_node,
2278                               &dma->channels);
2279         }
2280 }
2281
2282 static int __init d40_dmaengine_init(struct d40_base *base,
2283                                      int num_reserved_chans)
2284 {
2285         int err ;
2286
2287         d40_chan_init(base, &base->dma_slave, base->log_chans,
2288                       0, base->num_log_chans);
2289
2290         dma_cap_zero(base->dma_slave.cap_mask);
2291         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2292
2293         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2294         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2295         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2296         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2297         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2298         base->dma_slave.device_tx_status = d40_tx_status;
2299         base->dma_slave.device_issue_pending = d40_issue_pending;
2300         base->dma_slave.device_control = d40_control;
2301         base->dma_slave.dev = base->dev;
2302
2303         err = dma_async_device_register(&base->dma_slave);
2304
2305         if (err) {
2306                 d40_err(base->dev, "Failed to register slave channels\n");
2307                 goto failure1;
2308         }
2309
2310         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2311                       base->num_log_chans, base->plat_data->memcpy_len);
2312
2313         dma_cap_zero(base->dma_memcpy.cap_mask);
2314         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2315         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2316
2317         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2318         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2319         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2320         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2321         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2322         base->dma_memcpy.device_tx_status = d40_tx_status;
2323         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2324         base->dma_memcpy.device_control = d40_control;
2325         base->dma_memcpy.dev = base->dev;
2326         /*
2327          * This controller can only access address at even
2328          * 32bit boundaries, i.e. 2^2
2329          */
2330         base->dma_memcpy.copy_align = 2;
2331
2332         err = dma_async_device_register(&base->dma_memcpy);
2333
2334         if (err) {
2335                 d40_err(base->dev,
2336                         "Failed to regsiter memcpy only channels\n");
2337                 goto failure2;
2338         }
2339
2340         d40_chan_init(base, &base->dma_both, base->phy_chans,
2341                       0, num_reserved_chans);
2342
2343         dma_cap_zero(base->dma_both.cap_mask);
2344         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2345         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2346         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2347
2348         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2349         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2350         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2351         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2352         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2353         base->dma_both.device_tx_status = d40_tx_status;
2354         base->dma_both.device_issue_pending = d40_issue_pending;
2355         base->dma_both.device_control = d40_control;
2356         base->dma_both.dev = base->dev;
2357         base->dma_both.copy_align = 2;
2358         err = dma_async_device_register(&base->dma_both);
2359
2360         if (err) {
2361                 d40_err(base->dev,
2362                         "Failed to register logical and physical capable channels\n");
2363                 goto failure3;
2364         }
2365         return 0;
2366 failure3:
2367         dma_async_device_unregister(&base->dma_memcpy);
2368 failure2:
2369         dma_async_device_unregister(&base->dma_slave);
2370 failure1:
2371         return err;
2372 }
2373
2374 /* Initialization functions. */
2375
2376 static int __init d40_phy_res_init(struct d40_base *base)
2377 {
2378         int i;
2379         int num_phy_chans_avail = 0;
2380         u32 val[2];
2381         int odd_even_bit = -2;
2382
2383         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2384         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2385
2386         for (i = 0; i < base->num_phy_chans; i++) {
2387                 base->phy_res[i].num = i;
2388                 odd_even_bit += 2 * ((i % 2) == 0);
2389                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2390                         /* Mark security only channels as occupied */
2391                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2392                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2393                 } else {
2394                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2395                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2396                         num_phy_chans_avail++;
2397                 }
2398                 spin_lock_init(&base->phy_res[i].lock);
2399         }
2400
2401         /* Mark disabled channels as occupied */
2402         for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2403                 int chan = base->plat_data->disabled_channels[i];
2404
2405                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2406                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2407                 num_phy_chans_avail--;
2408         }
2409
2410         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2411                  num_phy_chans_avail, base->num_phy_chans);
2412
2413         /* Verify settings extended vs standard */
2414         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2415
2416         for (i = 0; i < base->num_phy_chans; i++) {
2417
2418                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2419                     (val[0] & 0x3) != 1)
2420                         dev_info(base->dev,
2421                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2422                                  __func__, i, val[0] & 0x3);
2423
2424                 val[0] = val[0] >> 2;
2425         }
2426
2427         return num_phy_chans_avail;
2428 }
2429
2430 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2431 {
2432         static const struct d40_reg_val dma_id_regs[] = {
2433                 /* Peripheral Id */
2434                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2435                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2436                 /*
2437                  * D40_DREG_PERIPHID2 Depends on HW revision:
2438                  *  DB8500ed has 0x0008,
2439                  *  ? has 0x0018,
2440                  *  DB8500v1 has 0x0028
2441                  *  DB8500v2 has 0x0038
2442                  */
2443                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2444
2445                 /* PCell Id */
2446                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2447                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2448                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2449                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2450         };
2451         struct stedma40_platform_data *plat_data;
2452         struct clk *clk = NULL;
2453         void __iomem *virtbase = NULL;
2454         struct resource *res = NULL;
2455         struct d40_base *base = NULL;
2456         int num_log_chans = 0;
2457         int num_phy_chans;
2458         int i;
2459         u32 val;
2460         u32 rev;
2461
2462         clk = clk_get(&pdev->dev, NULL);
2463
2464         if (IS_ERR(clk)) {
2465                 d40_err(&pdev->dev, "No matching clock found\n");
2466                 goto failure;
2467         }
2468
2469         clk_enable(clk);
2470
2471         /* Get IO for DMAC base address */
2472         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2473         if (!res)
2474                 goto failure;
2475
2476         if (request_mem_region(res->start, resource_size(res),
2477                                D40_NAME " I/O base") == NULL)
2478                 goto failure;
2479
2480         virtbase = ioremap(res->start, resource_size(res));
2481         if (!virtbase)
2482                 goto failure;
2483
2484         /* HW version check */
2485         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2486                 if (dma_id_regs[i].val !=
2487                     readl(virtbase + dma_id_regs[i].reg)) {
2488                         d40_err(&pdev->dev,
2489                                 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2490                                 dma_id_regs[i].val,
2491                                 dma_id_regs[i].reg,
2492                                 readl(virtbase + dma_id_regs[i].reg));
2493                         goto failure;
2494                 }
2495         }
2496
2497         /* Get silicon revision and designer */
2498         val = readl(virtbase + D40_DREG_PERIPHID2);
2499
2500         if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2501             D40_HW_DESIGNER) {
2502                 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2503                         val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2504                         D40_HW_DESIGNER);
2505                 goto failure;
2506         }
2507
2508         rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2509                 D40_DREG_PERIPHID2_REV_POS;
2510
2511         /* The number of physical channels on this HW */
2512         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2513
2514         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2515                  rev, res->start);
2516
2517         plat_data = pdev->dev.platform_data;
2518
2519         /* Count the number of logical channels in use */
2520         for (i = 0; i < plat_data->dev_len; i++)
2521                 if (plat_data->dev_rx[i] != 0)
2522                         num_log_chans++;
2523
2524         for (i = 0; i < plat_data->dev_len; i++)
2525                 if (plat_data->dev_tx[i] != 0)
2526                         num_log_chans++;
2527
2528         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2529                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2530                        sizeof(struct d40_chan), GFP_KERNEL);
2531
2532         if (base == NULL) {
2533                 d40_err(&pdev->dev, "Out of memory\n");
2534                 goto failure;
2535         }
2536
2537         base->rev = rev;
2538         base->clk = clk;
2539         base->num_phy_chans = num_phy_chans;
2540         base->num_log_chans = num_log_chans;
2541         base->phy_start = res->start;
2542         base->phy_size = resource_size(res);
2543         base->virtbase = virtbase;
2544         base->plat_data = plat_data;
2545         base->dev = &pdev->dev;
2546         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2547         base->log_chans = &base->phy_chans[num_phy_chans];
2548
2549         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2550                                 GFP_KERNEL);
2551         if (!base->phy_res)
2552                 goto failure;
2553
2554         base->lookup_phy_chans = kzalloc(num_phy_chans *
2555                                          sizeof(struct d40_chan *),
2556                                          GFP_KERNEL);
2557         if (!base->lookup_phy_chans)
2558                 goto failure;
2559
2560         if (num_log_chans + plat_data->memcpy_len) {
2561                 /*
2562                  * The max number of logical channels are event lines for all
2563                  * src devices and dst devices
2564                  */
2565                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2566                                                  sizeof(struct d40_chan *),
2567                                                  GFP_KERNEL);
2568                 if (!base->lookup_log_chans)
2569                         goto failure;
2570         }
2571
2572         base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2573                                             sizeof(struct d40_desc *) *
2574                                             D40_LCLA_LINK_PER_EVENT_GRP,
2575                                             GFP_KERNEL);
2576         if (!base->lcla_pool.alloc_map)
2577                 goto failure;
2578
2579         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2580                                             0, SLAB_HWCACHE_ALIGN,
2581                                             NULL);
2582         if (base->desc_slab == NULL)
2583                 goto failure;
2584
2585         return base;
2586
2587 failure:
2588         if (!IS_ERR(clk)) {
2589                 clk_disable(clk);
2590                 clk_put(clk);
2591         }
2592         if (virtbase)
2593                 iounmap(virtbase);
2594         if (res)
2595                 release_mem_region(res->start,
2596                                    resource_size(res));
2597         if (virtbase)
2598                 iounmap(virtbase);
2599
2600         if (base) {
2601                 kfree(base->lcla_pool.alloc_map);
2602                 kfree(base->lookup_log_chans);
2603                 kfree(base->lookup_phy_chans);
2604                 kfree(base->phy_res);
2605                 kfree(base);
2606         }
2607
2608         return NULL;
2609 }
2610
2611 static void __init d40_hw_init(struct d40_base *base)
2612 {
2613
2614         static const struct d40_reg_val dma_init_reg[] = {
2615                 /* Clock every part of the DMA block from start */
2616                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2617
2618                 /* Interrupts on all logical channels */
2619                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2620                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2621                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2622                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2623                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2624                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2625                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2626                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2627                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2628                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2629                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2630                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2631         };
2632         int i;
2633         u32 prmseo[2] = {0, 0};
2634         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2635         u32 pcmis = 0;
2636         u32 pcicr = 0;
2637
2638         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2639                 writel(dma_init_reg[i].val,
2640                        base->virtbase + dma_init_reg[i].reg);
2641
2642         /* Configure all our dma channels to default settings */
2643         for (i = 0; i < base->num_phy_chans; i++) {
2644
2645                 activeo[i % 2] = activeo[i % 2] << 2;
2646
2647                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2648                     == D40_ALLOC_PHY) {
2649                         activeo[i % 2] |= 3;
2650                         continue;
2651                 }
2652
2653                 /* Enable interrupt # */
2654                 pcmis = (pcmis << 1) | 1;
2655
2656                 /* Clear interrupt # */
2657                 pcicr = (pcicr << 1) | 1;
2658
2659                 /* Set channel to physical mode */
2660                 prmseo[i % 2] = prmseo[i % 2] << 2;
2661                 prmseo[i % 2] |= 1;
2662
2663         }
2664
2665         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2666         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2667         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2668         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2669
2670         /* Write which interrupt to enable */
2671         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2672
2673         /* Write which interrupt to clear */
2674         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2675
2676 }
2677
2678 static int __init d40_lcla_allocate(struct d40_base *base)
2679 {
2680         struct d40_lcla_pool *pool = &base->lcla_pool;
2681         unsigned long *page_list;
2682         int i, j;
2683         int ret = 0;
2684
2685         /*
2686          * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2687          * To full fill this hardware requirement without wasting 256 kb
2688          * we allocate pages until we get an aligned one.
2689          */
2690         page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2691                             GFP_KERNEL);
2692
2693         if (!page_list) {
2694                 ret = -ENOMEM;
2695                 goto failure;
2696         }
2697
2698         /* Calculating how many pages that are required */
2699         base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2700
2701         for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2702                 page_list[i] = __get_free_pages(GFP_KERNEL,
2703                                                 base->lcla_pool.pages);
2704                 if (!page_list[i]) {
2705
2706                         d40_err(base->dev, "Failed to allocate %d pages.\n",
2707                                 base->lcla_pool.pages);
2708
2709                         for (j = 0; j < i; j++)
2710                                 free_pages(page_list[j], base->lcla_pool.pages);
2711                         goto failure;
2712                 }
2713
2714                 if ((virt_to_phys((void *)page_list[i]) &
2715                      (LCLA_ALIGNMENT - 1)) == 0)
2716                         break;
2717         }
2718
2719         for (j = 0; j < i; j++)
2720                 free_pages(page_list[j], base->lcla_pool.pages);
2721
2722         if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2723                 base->lcla_pool.base = (void *)page_list[i];
2724         } else {
2725                 /*
2726                  * After many attempts and no succees with finding the correct
2727                  * alignment, try with allocating a big buffer.
2728                  */
2729                 dev_warn(base->dev,
2730                          "[%s] Failed to get %d pages @ 18 bit align.\n",
2731                          __func__, base->lcla_pool.pages);
2732                 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2733                                                          base->num_phy_chans +
2734                                                          LCLA_ALIGNMENT,
2735                                                          GFP_KERNEL);
2736                 if (!base->lcla_pool.base_unaligned) {
2737                         ret = -ENOMEM;
2738                         goto failure;
2739                 }
2740
2741                 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2742                                                  LCLA_ALIGNMENT);
2743         }
2744
2745         pool->dma_addr = dma_map_single(base->dev, pool->base,
2746                                         SZ_1K * base->num_phy_chans,
2747                                         DMA_TO_DEVICE);
2748         if (dma_mapping_error(base->dev, pool->dma_addr)) {
2749                 pool->dma_addr = 0;
2750                 ret = -ENOMEM;
2751                 goto failure;
2752         }
2753
2754         writel(virt_to_phys(base->lcla_pool.base),
2755                base->virtbase + D40_DREG_LCLA);
2756 failure:
2757         kfree(page_list);
2758         return ret;
2759 }
2760
2761 static int __init d40_probe(struct platform_device *pdev)
2762 {
2763         int err;
2764         int ret = -ENOENT;
2765         struct d40_base *base;
2766         struct resource *res = NULL;
2767         int num_reserved_chans;
2768         u32 val;
2769
2770         base = d40_hw_detect_init(pdev);
2771
2772         if (!base)
2773                 goto failure;
2774
2775         num_reserved_chans = d40_phy_res_init(base);
2776
2777         platform_set_drvdata(pdev, base);
2778
2779         spin_lock_init(&base->interrupt_lock);
2780         spin_lock_init(&base->execmd_lock);
2781
2782         /* Get IO for logical channel parameter address */
2783         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2784         if (!res) {
2785                 ret = -ENOENT;
2786                 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
2787                 goto failure;
2788         }
2789         base->lcpa_size = resource_size(res);
2790         base->phy_lcpa = res->start;
2791
2792         if (request_mem_region(res->start, resource_size(res),
2793                                D40_NAME " I/O lcpa") == NULL) {
2794                 ret = -EBUSY;
2795                 d40_err(&pdev->dev,
2796                         "Failed to request LCPA region 0x%x-0x%x\n",
2797                         res->start, res->end);
2798                 goto failure;
2799         }
2800
2801         /* We make use of ESRAM memory for this. */
2802         val = readl(base->virtbase + D40_DREG_LCPA);
2803         if (res->start != val && val != 0) {
2804                 dev_warn(&pdev->dev,
2805                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2806                          __func__, val, res->start);
2807         } else
2808                 writel(res->start, base->virtbase + D40_DREG_LCPA);
2809
2810         base->lcpa_base = ioremap(res->start, resource_size(res));
2811         if (!base->lcpa_base) {
2812                 ret = -ENOMEM;
2813                 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
2814                 goto failure;
2815         }
2816
2817         ret = d40_lcla_allocate(base);
2818         if (ret) {
2819                 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
2820                 goto failure;
2821         }
2822
2823         spin_lock_init(&base->lcla_pool.lock);
2824
2825         base->irq = platform_get_irq(pdev, 0);
2826
2827         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2828         if (ret) {
2829                 d40_err(&pdev->dev, "No IRQ defined\n");
2830                 goto failure;
2831         }
2832
2833         err = d40_dmaengine_init(base, num_reserved_chans);
2834         if (err)
2835                 goto failure;
2836
2837         d40_hw_init(base);
2838
2839         dev_info(base->dev, "initialized\n");
2840         return 0;
2841
2842 failure:
2843         if (base) {
2844                 if (base->desc_slab)
2845                         kmem_cache_destroy(base->desc_slab);
2846                 if (base->virtbase)
2847                         iounmap(base->virtbase);
2848
2849                 if (base->lcla_pool.dma_addr)
2850                         dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2851                                          SZ_1K * base->num_phy_chans,
2852                                          DMA_TO_DEVICE);
2853
2854                 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2855                         free_pages((unsigned long)base->lcla_pool.base,
2856                                    base->lcla_pool.pages);
2857
2858                 kfree(base->lcla_pool.base_unaligned);
2859
2860                 if (base->phy_lcpa)
2861                         release_mem_region(base->phy_lcpa,
2862                                            base->lcpa_size);
2863                 if (base->phy_start)
2864                         release_mem_region(base->phy_start,
2865                                            base->phy_size);
2866                 if (base->clk) {
2867                         clk_disable(base->clk);
2868                         clk_put(base->clk);
2869                 }
2870
2871                 kfree(base->lcla_pool.alloc_map);
2872                 kfree(base->lookup_log_chans);
2873                 kfree(base->lookup_phy_chans);
2874                 kfree(base->phy_res);
2875                 kfree(base);
2876         }
2877
2878         d40_err(&pdev->dev, "probe failed\n");
2879         return ret;
2880 }
2881
2882 static struct platform_driver d40_driver = {
2883         .driver = {
2884                 .owner = THIS_MODULE,
2885                 .name  = D40_NAME,
2886         },
2887 };
2888
2889 static int __init stedma40_init(void)
2890 {
2891         return platform_driver_probe(&d40_driver, d40_probe);
2892 }
2893 arch_initcall(stedma40_init);