drivers/dma/ste_dma40.c

   1 /*
   2  * Copyright (C) Ericsson AB 2007-2008
   3  * Copyright (C) ST-Ericsson SA 2008-2010
   4  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
   5  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
   6  * License terms: GNU General Public License (GPL) version 2
   7  */
   8
   9 #include <linux/kernel.h>
  10 #include <linux/slab.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/clk.h>
  14 #include <linux/delay.h>
  15 #include <linux/err.h>
  16
  17 #include <plat/ste_dma40.h>
  18
  19 #include "ste_dma40_ll.h"
  20
  21 #define D40_NAME "dma40"
  22
  23 #define D40_PHY_CHAN -1
  24
  25 /* For masking out/in 2 bit channel positions */
  26 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
  27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  28
  29 /* Maximum iterations taken before giving up suspending a channel */
  30 #define D40_SUSPEND_MAX_IT 500
  31
  32 /* Hardware requirement on LCLA alignment */
  33 #define LCLA_ALIGNMENT 0x40000
  34
  35 /* Max number of links per event group */
  36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
  37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  38
  39 /* Attempts before giving up to trying to get pages that are aligned */
  40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
  41
  42 /* Bit markings for allocation map */
  43 #define D40_ALLOC_FREE          (1 << 31)
  44 #define D40_ALLOC_PHY           (1 << 30)
  45 #define D40_ALLOC_LOG_FREE      0
  46
  47 /* Hardware designer of the block */
  48 #define D40_HW_DESIGNER 0x8
  49
  50 /**
  51  * enum 40_command - The different commands and/or statuses.
  52  *
  53  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  54  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  55  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  56  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  57  */
  58 enum d40_command {
  59         D40_DMA_STOP            = 0,
  60         D40_DMA_RUN             = 1,
  61         D40_DMA_SUSPEND_REQ     = 2,
  62         D40_DMA_SUSPENDED       = 3
  63 };
  64
  65 /**
  66  * struct d40_lli_pool - Structure for keeping LLIs in memory
  67  *
  68  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  69  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  70  * pre_alloc_lli is used.
  71  * @dma_addr: DMA address, if mapped
  72  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  73  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  74  * one buffer to one buffer.
  75  */
  76 struct d40_lli_pool {
  77         void    *base;
  78         int      size;
  79         dma_addr_t      dma_addr;
  80         /* Space for dst and src, plus an extra for padding */
  81         u8       pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  82 };
  83
  84 /**
  85  * struct d40_desc - A descriptor is one DMA job.
  86  *
  87  * @lli_phy: LLI settings for physical channel. Both src and dst=
  88  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  89  * lli_len equals one.
  90  * @lli_log: Same as above but for logical channels.
  91  * @lli_pool: The pool with two entries pre-allocated.
  92  * @lli_len: Number of llis of current descriptor.
  93  * @lli_current: Number of transfered llis.
  94  * @lcla_alloc: Number of LCLA entries allocated.
  95  * @txd: DMA engine struct. Used for among other things for communication
  96  * during a transfer.
  97  * @node: List entry.
  98  * @is_in_client_list: true if the client owns this descriptor.
  99  * the previous one.
 100  *
 101  * This descriptor is used for both logical and physical transfers.
 102  */
 103 struct d40_desc {
 104         /* LLI physical */
 105         struct d40_phy_lli_bidir         lli_phy;
 106         /* LLI logical */
 107         struct d40_log_lli_bidir         lli_log;
 108
 109         struct d40_lli_pool              lli_pool;
 110         int                              lli_len;
 111         int                              lli_current;
 112         int                              lcla_alloc;
 113
 114         struct dma_async_tx_descriptor   txd;
 115         struct list_head                 node;
 116
 117         bool                             is_in_client_list;
 118 };
 119
 120 /**
 121  * struct d40_lcla_pool - LCLA pool settings and data.
 122  *
 123  * @base: The virtual address of LCLA. 18 bit aligned.
 124  * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
 125  * This pointer is only there for clean-up on error.
 126  * @pages: The number of pages needed for all physical channels.
 127  * Only used later for clean-up on error
 128  * @lock: Lock to protect the content in this struct.
 129  * @alloc_map: big map over which LCLA entry is own by which job.
 130  */
 131 struct d40_lcla_pool {
 132         void            *base;
 133         dma_addr_t      dma_addr;
 134         void            *base_unaligned;
 135         int              pages;
 136         spinlock_t       lock;
 137         struct d40_desc **alloc_map;
 138 };
 139
 140 /**
 141  * struct d40_phy_res - struct for handling eventlines mapped to physical
 142  * channels.
 143  *
 144  * @lock: A lock protection this entity.
 145  * @num: The physical channel number of this entity.
 146  * @allocated_src: Bit mapped to show which src event line's are mapped to
 147  * this physical channel. Can also be free or physically allocated.
 148  * @allocated_dst: Same as for src but is dst.
 149  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
 150  * event line number.
 151  */
 152 struct d40_phy_res {
 153         spinlock_t lock;
 154         int        num;
 155         u32        allocated_src;
 156         u32        allocated_dst;
 157 };
 158
 159 struct d40_base;
 160
 161 /**
 162  * struct d40_chan - Struct that describes a channel.
 163  *
 164  * @lock: A spinlock to protect this struct.
 165  * @log_num: The logical number, if any of this channel.
 166  * @completed: Starts with 1, after first interrupt it is set to dma engine's
 167  * current cookie.
 168  * @pending_tx: The number of pending transfers. Used between interrupt handler
 169  * and tasklet.
 170  * @busy: Set to true when transfer is ongoing on this channel.
 171  * @phy_chan: Pointer to physical channel which this instance runs on. If this
 172  * point is NULL, then the channel is not allocated.
 173  * @chan: DMA engine handle.
 174  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
 175  * transfer and call client callback.
 176  * @client: Cliented owned descriptor list.
 177  * @active: Active descriptor.
 178  * @queue: Queued jobs.
 179  * @dma_cfg: The client configuration of this dma channel.
 180  * @configured: whether the dma_cfg configuration is valid
 181  * @base: Pointer to the device instance struct.
 182  * @src_def_cfg: Default cfg register setting for src.
 183  * @dst_def_cfg: Default cfg register setting for dst.
 184  * @log_def: Default logical channel settings.
 185  * @lcla: Space for one dst src pair for logical channel transfers.
 186  * @lcpa: Pointer to dst and src lcpa settings.
 187  *
 188  * This struct can either "be" a logical or a physical channel.
 189  */
 190 struct d40_chan {
 191         spinlock_t                       lock;
 192         int                              log_num;
 193         /* ID of the most recent completed transfer */
 194         int                              completed;
 195         int                              pending_tx;
 196         bool                             busy;
 197         struct d40_phy_res              *phy_chan;
 198         struct dma_chan                  chan;
 199         struct tasklet_struct            tasklet;
 200         struct list_head                 client;
 201         struct list_head                 active;
 202         struct list_head                 queue;
 203         struct stedma40_chan_cfg         dma_cfg;
 204         bool                             configured;
 205         struct d40_base                 *base;
 206         /* Default register configurations */
 207         u32                              src_def_cfg;
 208         u32                              dst_def_cfg;
 209         struct d40_def_lcsp              log_def;
 210         struct d40_log_lli_full         *lcpa;
 211         /* Runtime reconfiguration */
 212         dma_addr_t                      runtime_addr;
 213         enum dma_data_direction         runtime_direction;
 214 };
 215
 216 /**
 217  * struct d40_base - The big global struct, one for each probe'd instance.
 218  *
 219  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
 220  * @execmd_lock: Lock for execute command usage since several channels share
 221  * the same physical register.
 222  * @dev: The device structure.
 223  * @virtbase: The virtual base address of the DMA's register.
 224  * @rev: silicon revision detected.
 225  * @clk: Pointer to the DMA clock structure.
 226  * @phy_start: Physical memory start of the DMA registers.
 227  * @phy_size: Size of the DMA register map.
 228  * @irq: The IRQ number.
 229  * @num_phy_chans: The number of physical channels. Read from HW. This
 230  * is the number of available channels for this driver, not counting "Secure
 231  * mode" allocated physical channels.
 232  * @num_log_chans: The number of logical channels. Calculated from
 233  * num_phy_chans.
 234  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
 235  * @dma_slave: dma_device channels that can do only do slave transfers.
 236  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
 237  * @log_chans: Room for all possible logical channels in system.
 238  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
 239  * to log_chans entries.
 240  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
 241  * to phy_chans entries.
 242  * @plat_data: Pointer to provided platform_data which is the driver
 243  * configuration.
 244  * @phy_res: Vector containing all physical channels.
 245  * @lcla_pool: lcla pool settings and data.
 246  * @lcpa_base: The virtual mapped address of LCPA.
 247  * @phy_lcpa: The physical address of the LCPA.
 248  * @lcpa_size: The size of the LCPA area.
 249  * @desc_slab: cache for descriptors.
 250  */
 251 struct d40_base {
 252         spinlock_t                       interrupt_lock;
 253         spinlock_t                       execmd_lock;
 254         struct device                    *dev;
 255         void __iomem                     *virtbase;
 256         u8                                rev:4;
 257         struct clk                       *clk;
 258         phys_addr_t                       phy_start;
 259         resource_size_t                   phy_size;
 260         int                               irq;
 261         int                               num_phy_chans;
 262         int                               num_log_chans;
 263         struct dma_device                 dma_both;
 264         struct dma_device                 dma_slave;
 265         struct dma_device                 dma_memcpy;
 266         struct d40_chan                  *phy_chans;
 267         struct d40_chan                  *log_chans;
 268         struct d40_chan                 **lookup_log_chans;
 269         struct d40_chan                 **lookup_phy_chans;
 270         struct stedma40_platform_data    *plat_data;
 271         /* Physical half channels */
 272         struct d40_phy_res               *phy_res;
 273         struct d40_lcla_pool              lcla_pool;
 274         void                             *lcpa_base;
 275         dma_addr_t                        phy_lcpa;
 276         resource_size_t                   lcpa_size;
 277         struct kmem_cache                *desc_slab;
 278 };
 279
 280 /**
 281  * struct d40_interrupt_lookup - lookup table for interrupt handler
 282  *
 283  * @src: Interrupt mask register.
 284  * @clr: Interrupt clear register.
 285  * @is_error: true if this is an error interrupt.
 286  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
 287  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
 288  */
 289 struct d40_interrupt_lookup {
 290         u32 src;
 291         u32 clr;
 292         bool is_error;
 293         int offset;
 294 };
 295
 296 /**
 297  * struct d40_reg_val - simple lookup struct
 298  *
 299  * @reg: The register.
 300  * @val: The value that belongs to the register in reg.
 301  */
 302 struct d40_reg_val {
 303         unsigned int reg;
 304         unsigned int val;
 305 };
 306
 307 static struct device *chan2dev(struct d40_chan *d40c)
 308 {
 309         return &d40c->chan.dev->device;
 310 }
 311
 312 static bool chan_is_physical(struct d40_chan *chan)
 313 {
 314         return chan->log_num == D40_PHY_CHAN;
 315 }
 316
 317 static bool chan_is_logical(struct d40_chan *chan)
 318 {
 319         return !chan_is_physical(chan);
 320 }
 321
 322 static void __iomem *chan_base(struct d40_chan *chan)
 323 {
 324         return chan->base->virtbase + D40_DREG_PCBASE +
 325                chan->phy_chan->num * D40_DREG_PCDELTA;
 326 }
 327
 328 #define d40_err(dev, format, arg...)            \
 329         dev_err(dev, "[%s] " format, __func__, ## arg)
 330
 331 #define chan_err(d40c, format, arg...)          \
 332         d40_err(chan2dev(d40c), format, ## arg)
 333
 334 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
 335                               int lli_len, bool is_log)
 336 {
 337         u32 align;
 338         void *base;
 339
 340         if (is_log)
 341                 align = sizeof(struct d40_log_lli);
 342         else
 343                 align = sizeof(struct d40_phy_lli);
 344
 345         if (lli_len == 1) {
 346                 base = d40d->lli_pool.pre_alloc_lli;
 347                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 348                 d40d->lli_pool.base = NULL;
 349         } else {
 350                 d40d->lli_pool.size = lli_len * 2 * align;
 351
 352                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 353                 d40d->lli_pool.base = base;
 354
 355                 if (d40d->lli_pool.base == NULL)
 356                         return -ENOMEM;
 357         }
 358
 359         if (is_log) {
 360                 d40d->lli_log.src = PTR_ALIGN(base, align);
 361                 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
 362
 363                 d40d->lli_pool.dma_addr = 0;
 364         } else {
 365                 d40d->lli_phy.src = PTR_ALIGN(base, align);
 366                 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
 367
 368                 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
 369                                                          d40d->lli_phy.src,
 370                                                          d40d->lli_pool.size,
 371                                                          DMA_TO_DEVICE);
 372
 373                 if (dma_mapping_error(d40c->base->dev,
 374                                       d40d->lli_pool.dma_addr)) {
 375                         kfree(d40d->lli_pool.base);
 376                         d40d->lli_pool.base = NULL;
 377                         d40d->lli_pool.dma_addr = 0;
 378                         return -ENOMEM;
 379                 }
 380         }
 381
 382         return 0;
 383 }
 384
 385 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
 386 {
 387         if (d40d->lli_pool.dma_addr)
 388                 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
 389                                  d40d->lli_pool.size, DMA_TO_DEVICE);
 390
 391         kfree(d40d->lli_pool.base);
 392         d40d->lli_pool.base = NULL;
 393         d40d->lli_pool.size = 0;
 394         d40d->lli_log.src = NULL;
 395         d40d->lli_log.dst = NULL;
 396         d40d->lli_phy.src = NULL;
 397         d40d->lli_phy.dst = NULL;
 398 }
 399
 400 static int d40_lcla_alloc_one(struct d40_chan *d40c,
 401                               struct d40_desc *d40d)
 402 {
 403         unsigned long flags;
 404         int i;
 405         int ret = -EINVAL;
 406         int p;
 407
 408         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 409
 410         p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
 411
 412         /*
 413          * Allocate both src and dst at the same time, therefore the half
 414          * start on 1 since 0 can't be used since zero is used as end marker.
 415          */
 416         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 417                 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
 418                         d40c->base->lcla_pool.alloc_map[p + i] = d40d;
 419                         d40d->lcla_alloc++;
 420                         ret = i;
 421                         break;
 422                 }
 423         }
 424
 425         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 426
 427         return ret;
 428 }
 429
 430 static int d40_lcla_free_all(struct d40_chan *d40c,
 431                              struct d40_desc *d40d)
 432 {
 433         unsigned long flags;
 434         int i;
 435         int ret = -EINVAL;
 436
 437         if (chan_is_physical(d40c))
 438                 return 0;
 439
 440         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 441
 442         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 443                 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 444                                                     D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
 445                         d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 446                                                         D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
 447                         d40d->lcla_alloc--;
 448                         if (d40d->lcla_alloc == 0) {
 449                                 ret = 0;
 450                                 break;
 451                         }
 452                 }
 453         }
 454
 455         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 456
 457         return ret;
 458
 459 }
 460
 461 static void d40_desc_remove(struct d40_desc *d40d)
 462 {
 463         list_del(&d40d->node);
 464 }
 465
 466 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 467 {
 468         struct d40_desc *desc = NULL;
 469
 470         if (!list_empty(&d40c->client)) {
 471                 struct d40_desc *d;
 472                 struct d40_desc *_d;
 473
 474                 list_for_each_entry_safe(d, _d, &d40c->client, node)
 475                         if (async_tx_test_ack(&d->txd)) {
 476                                 d40_pool_lli_free(d40c, d);
 477                                 d40_desc_remove(d);
 478                                 desc = d;
 479                                 memset(desc, 0, sizeof(*desc));
 480                                 break;
 481                         }
 482         }
 483
 484         if (!desc)
 485                 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
 486
 487         if (desc)
 488                 INIT_LIST_HEAD(&desc->node);
 489
 490         return desc;
 491 }
 492
 493 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 494 {
 495
 496         d40_pool_lli_free(d40c, d40d);
 497         d40_lcla_free_all(d40c, d40d);
 498         kmem_cache_free(d40c->base->desc_slab, d40d);
 499 }
 500
 501 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 502 {
 503         list_add_tail(&desc->node, &d40c->active);
 504 }
 505
 506 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 507 {
 508         int curr_lcla = -EINVAL, next_lcla;
 509
 510         if (chan_is_physical(d40c)) {
 511                 d40_phy_lli_write(d40c->base->virtbase,
 512                                   d40c->phy_chan->num,
 513                                   d40d->lli_phy.dst,
 514                                   d40d->lli_phy.src);
 515                 d40d->lli_current = d40d->lli_len;
 516         } else {
 517
 518                 if ((d40d->lli_len - d40d->lli_current) > 1)
 519                         curr_lcla = d40_lcla_alloc_one(d40c, d40d);
 520
 521                 d40_log_lli_lcpa_write(d40c->lcpa,
 522                                        &d40d->lli_log.dst[d40d->lli_current],
 523                                        &d40d->lli_log.src[d40d->lli_current],
 524                                        curr_lcla);
 525
 526                 d40d->lli_current++;
 527                 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
 528                         unsigned int lcla_offset = d40c->phy_chan->num * 1024 +
 529                                                    8 * curr_lcla * 2;
 530                         struct d40_lcla_pool *pool = &d40c->base->lcla_pool;
 531                         struct d40_log_lli *lcla = pool->base + lcla_offset;
 532
 533                         if (d40d->lli_current + 1 < d40d->lli_len)
 534                                 next_lcla = d40_lcla_alloc_one(d40c, d40d);
 535                         else
 536                                 next_lcla = -EINVAL;
 537
 538                         d40_log_lli_lcla_write(lcla,
 539                                                &d40d->lli_log.dst[d40d->lli_current],
 540                                                &d40d->lli_log.src[d40d->lli_current],
 541                                                next_lcla);
 542
 543                         dma_sync_single_range_for_device(d40c->base->dev,
 544                                                 pool->dma_addr, lcla_offset,
 545                                                 2 * sizeof(struct d40_log_lli),
 546                                                 DMA_TO_DEVICE);
 547
 548                         curr_lcla = next_lcla;
 549
 550                         if (curr_lcla == -EINVAL) {
 551                                 d40d->lli_current++;
 552                                 break;
 553                         }
 554
 555                 }
 556         }
 557 }
 558
 559 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 560 {
 561         struct d40_desc *d;
 562
 563         if (list_empty(&d40c->active))
 564                 return NULL;
 565
 566         d = list_first_entry(&d40c->active,
 567                              struct d40_desc,
 568                              node);
 569         return d;
 570 }
 571
 572 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 573 {
 574         list_add_tail(&desc->node, &d40c->queue);
 575 }
 576
 577 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 578 {
 579         struct d40_desc *d;
 580
 581         if (list_empty(&d40c->queue))
 582                 return NULL;
 583
 584         d = list_first_entry(&d40c->queue,
 585                              struct d40_desc,
 586                              node);
 587         return d;
 588 }
 589
 590 static int d40_psize_2_burst_size(bool is_log, int psize)
 591 {
 592         if (is_log) {
 593                 if (psize == STEDMA40_PSIZE_LOG_1)
 594                         return 1;
 595         } else {
 596                 if (psize == STEDMA40_PSIZE_PHY_1)
 597                         return 1;
 598         }
 599
 600         return 2 << psize;
 601 }
 602
 603 /*
 604  * The dma only supports transmitting packages up to
 605  * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
 606  * dma elements required to send the entire sg list
 607  */
 608 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
 609 {
 610         int dmalen;
 611         u32 max_w = max(data_width1, data_width2);
 612         u32 min_w = min(data_width1, data_width2);
 613         u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
 614
 615         if (seg_max > STEDMA40_MAX_SEG_SIZE)
 616                 seg_max -= (1 << max_w);
 617
 618         if (!IS_ALIGNED(size, 1 << max_w))
 619                 return -EINVAL;
 620
 621         if (size <= seg_max)
 622                 dmalen = 1;
 623         else {
 624                 dmalen = size / seg_max;
 625                 if (dmalen * seg_max < size)
 626                         dmalen++;
 627         }
 628         return dmalen;
 629 }
 630
 631 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
 632                            u32 data_width1, u32 data_width2)
 633 {
 634         struct scatterlist *sg;
 635         int i;
 636         int len = 0;
 637         int ret;
 638
 639         for_each_sg(sgl, sg, sg_len, i) {
 640                 ret = d40_size_2_dmalen(sg_dma_len(sg),
 641                                         data_width1, data_width2);
 642                 if (ret < 0)
 643                         return ret;
 644                 len += ret;
 645         }
 646         return len;
 647 }
 648
 649 /* Support functions for logical channels */
 650
 651 static int d40_channel_execute_command(struct d40_chan *d40c,
 652                                        enum d40_command command)
 653 {
 654         u32 status;
 655         int i;
 656         void __iomem *active_reg;
 657         int ret = 0;
 658         unsigned long flags;
 659         u32 wmask;
 660
 661         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 662
 663         if (d40c->phy_chan->num % 2 == 0)
 664                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 665         else
 666                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 667
 668         if (command == D40_DMA_SUSPEND_REQ) {
 669                 status = (readl(active_reg) &
 670                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 671                         D40_CHAN_POS(d40c->phy_chan->num);
 672
 673                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 674                         goto done;
 675         }
 676
 677         wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
 678         writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
 679                active_reg);
 680
 681         if (command == D40_DMA_SUSPEND_REQ) {
 682
 683                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 684                         status = (readl(active_reg) &
 685                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 686                                 D40_CHAN_POS(d40c->phy_chan->num);
 687
 688                         cpu_relax();
 689                         /*
 690                          * Reduce the number of bus accesses while
 691                          * waiting for the DMA to suspend.
 692                          */
 693                         udelay(3);
 694
 695                         if (status == D40_DMA_STOP ||
 696                             status == D40_DMA_SUSPENDED)
 697                                 break;
 698                 }
 699
 700                 if (i == D40_SUSPEND_MAX_IT) {
 701                         chan_err(d40c,
 702                                 "unable to suspend the chl %d (log: %d) status %x\n",
 703                                 d40c->phy_chan->num, d40c->log_num,
 704                                 status);
 705                         dump_stack();
 706                         ret = -EBUSY;
 707                 }
 708
 709         }
 710 done:
 711         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 712         return ret;
 713 }
 714
 715 static void d40_term_all(struct d40_chan *d40c)
 716 {
 717         struct d40_desc *d40d;
 718
 719         /* Release active descriptors */
 720         while ((d40d = d40_first_active_get(d40c))) {
 721                 d40_desc_remove(d40d);
 722                 d40_desc_free(d40c, d40d);
 723         }
 724
 725         /* Release queued descriptors waiting for transfer */
 726         while ((d40d = d40_first_queued(d40c))) {
 727                 d40_desc_remove(d40d);
 728                 d40_desc_free(d40c, d40d);
 729         }
 730
 731
 732         d40c->pending_tx = 0;
 733         d40c->busy = false;
 734 }
 735
 736 static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
 737                                    u32 event, int reg)
 738 {
 739         void __iomem *addr = chan_base(d40c) + reg;
 740         int tries;
 741
 742         if (!enable) {
 743                 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 744                        | ~D40_EVENTLINE_MASK(event), addr);
 745                 return;
 746         }
 747
 748         /*
 749          * The hardware sometimes doesn't register the enable when src and dst
 750          * event lines are active on the same logical channel.  Retry to ensure
 751          * it does.  Usually only one retry is sufficient.
 752          */
 753         tries = 100;
 754         while (--tries) {
 755                 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 756                        | ~D40_EVENTLINE_MASK(event), addr);
 757
 758                 if (readl(addr) & D40_EVENTLINE_MASK(event))
 759                         break;
 760         }
 761
 762         if (tries != 99)
 763                 dev_dbg(chan2dev(d40c),
 764                         "[%s] workaround enable S%cLNK (%d tries)\n",
 765                         __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
 766                         100 - tries);
 767
 768         WARN_ON(!tries);
 769 }
 770
 771 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 772 {
 773         unsigned long flags;
 774
 775         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 776
 777         /* Enable event line connected to device (or memcpy) */
 778         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 779             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 780                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 781
 782                 __d40_config_set_event(d40c, do_enable, event,
 783                                        D40_CHAN_REG_SSLNK);
 784         }
 785
 786         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 787                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 788
 789                 __d40_config_set_event(d40c, do_enable, event,
 790                                        D40_CHAN_REG_SDLNK);
 791         }
 792
 793         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 794 }
 795
 796 static u32 d40_chan_has_events(struct d40_chan *d40c)
 797 {
 798         void __iomem *chanbase = chan_base(d40c);
 799         u32 val;
 800
 801         val = readl(chanbase + D40_CHAN_REG_SSLNK);
 802         val |= readl(chanbase + D40_CHAN_REG_SDLNK);
 803
 804         return val;
 805 }
 806
 807 static u32 d40_get_prmo(struct d40_chan *d40c)
 808 {
 809         static const unsigned int phy_map[] = {
 810                 [STEDMA40_PCHAN_BASIC_MODE]
 811                         = D40_DREG_PRMO_PCHAN_BASIC,
 812                 [STEDMA40_PCHAN_MODULO_MODE]
 813                         = D40_DREG_PRMO_PCHAN_MODULO,
 814                 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
 815                         = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
 816         };
 817         static const unsigned int log_map[] = {
 818                 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
 819                         = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
 820                 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
 821                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
 822                 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
 823                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
 824         };
 825
 826         if (chan_is_physical(d40c))
 827                 return phy_map[d40c->dma_cfg.mode_opt];
 828         else
 829                 return log_map[d40c->dma_cfg.mode_opt];
 830 }
 831
 832 static void d40_config_write(struct d40_chan *d40c)
 833 {
 834         u32 addr_base;
 835         u32 var;
 836
 837         /* Odd addresses are even addresses + 4 */
 838         addr_base = (d40c->phy_chan->num % 2) * 4;
 839         /* Setup channel mode to logical or physical */
 840         var = ((u32)(chan_is_logical(d40c)) + 1) <<
 841                 D40_CHAN_POS(d40c->phy_chan->num);
 842         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 843
 844         /* Setup operational mode option register */
 845         var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
 846
 847         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 848
 849         if (chan_is_logical(d40c)) {
 850                 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
 851                            & D40_SREG_ELEM_LOG_LIDX_MASK;
 852                 void __iomem *chanbase = chan_base(d40c);
 853
 854                 /* Set default config for CFG reg */
 855                 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
 856                 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
 857
 858                 /* Set LIDX for lcla */
 859                 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
 860                 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
 861         }
 862 }
 863
 864 static u32 d40_residue(struct d40_chan *d40c)
 865 {
 866         u32 num_elt;
 867
 868         if (chan_is_logical(d40c))
 869                 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
 870                         >> D40_MEM_LCSP2_ECNT_POS;
 871         else {
 872                 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
 873                 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
 874                           >> D40_SREG_ELEM_PHY_ECNT_POS;
 875         }
 876
 877         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
 878 }
 879
 880 static bool d40_tx_is_linked(struct d40_chan *d40c)
 881 {
 882         bool is_link;
 883
 884         if (chan_is_logical(d40c))
 885                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
 886         else
 887                 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
 888                           & D40_SREG_LNK_PHYS_LNK_MASK;
 889
 890         return is_link;
 891 }
 892
 893 static int d40_pause(struct dma_chan *chan)
 894 {
 895         struct d40_chan *d40c =
 896                 container_of(chan, struct d40_chan, chan);
 897         int res = 0;
 898         unsigned long flags;
 899
 900         if (!d40c->busy)
 901                 return 0;
 902
 903         spin_lock_irqsave(&d40c->lock, flags);
 904
 905         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 906         if (res == 0) {
 907                 if (chan_is_logical(d40c)) {
 908                         d40_config_set_event(d40c, false);
 909                         /* Resume the other logical channels if any */
 910                         if (d40_chan_has_events(d40c))
 911                                 res = d40_channel_execute_command(d40c,
 912                                                                   D40_DMA_RUN);
 913                 }
 914         }
 915
 916         spin_unlock_irqrestore(&d40c->lock, flags);
 917         return res;
 918 }
 919
 920 static int d40_resume(struct dma_chan *chan)
 921 {
 922         struct d40_chan *d40c =
 923                 container_of(chan, struct d40_chan, chan);
 924         int res = 0;
 925         unsigned long flags;
 926
 927         if (!d40c->busy)
 928                 return 0;
 929
 930         spin_lock_irqsave(&d40c->lock, flags);
 931
 932         if (d40c->base->rev == 0)
 933                 if (chan_is_logical(d40c)) {
 934                         res = d40_channel_execute_command(d40c,
 935                                                           D40_DMA_SUSPEND_REQ);
 936                         goto no_suspend;
 937                 }
 938
 939         /* If bytes left to transfer or linked tx resume job */
 940         if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
 941
 942                 if (chan_is_logical(d40c))
 943                         d40_config_set_event(d40c, true);
 944
 945                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 946         }
 947
 948 no_suspend:
 949         spin_unlock_irqrestore(&d40c->lock, flags);
 950         return res;
 951 }
 952
 953 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
 954 {
 955         struct d40_chan *d40c = container_of(tx->chan,
 956                                              struct d40_chan,
 957                                              chan);
 958         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
 959         unsigned long flags;
 960
 961         spin_lock_irqsave(&d40c->lock, flags);
 962
 963         d40c->chan.cookie++;
 964
 965         if (d40c->chan.cookie < 0)
 966                 d40c->chan.cookie = 1;
 967
 968         d40d->txd.cookie = d40c->chan.cookie;
 969
 970         d40_desc_queue(d40c, d40d);
 971
 972         spin_unlock_irqrestore(&d40c->lock, flags);
 973
 974         return tx->cookie;
 975 }
 976
 977 static int d40_start(struct d40_chan *d40c)
 978 {
 979         if (d40c->base->rev == 0) {
 980                 int err;
 981
 982                 if (chan_is_logical(d40c)) {
 983                         err = d40_channel_execute_command(d40c,
 984                                                           D40_DMA_SUSPEND_REQ);
 985                         if (err)
 986                                 return err;
 987                 }
 988         }
 989
 990         if (chan_is_logical(d40c))
 991                 d40_config_set_event(d40c, true);
 992
 993         return d40_channel_execute_command(d40c, D40_DMA_RUN);
 994 }
 995
 996 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
 997 {
 998         struct d40_desc *d40d;
 999         int err;
1000
1001         /* Start queued jobs, if any */
1002         d40d = d40_first_queued(d40c);
1003
1004         if (d40d != NULL) {
1005                 d40c->busy = true;
1006
1007                 /* Remove from queue */
1008                 d40_desc_remove(d40d);
1009
1010                 /* Add to active queue */
1011                 d40_desc_submit(d40c, d40d);
1012
1013                 /* Initiate DMA job */
1014                 d40_desc_load(d40c, d40d);
1015
1016                 /* Start dma job */
1017                 err = d40_start(d40c);
1018
1019                 if (err)
1020                         return NULL;
1021         }
1022
1023         return d40d;
1024 }
1025
1026 /* called from interrupt context */
1027 static void dma_tc_handle(struct d40_chan *d40c)
1028 {
1029         struct d40_desc *d40d;
1030
1031         /* Get first active entry from list */
1032         d40d = d40_first_active_get(d40c);
1033
1034         if (d40d == NULL)
1035                 return;
1036
1037         d40_lcla_free_all(d40c, d40d);
1038
1039         if (d40d->lli_current < d40d->lli_len) {
1040                 d40_desc_load(d40c, d40d);
1041                 /* Start dma job */
1042                 (void) d40_start(d40c);
1043                 return;
1044         }
1045
1046         if (d40_queue_start(d40c) == NULL)
1047                 d40c->busy = false;
1048
1049         d40c->pending_tx++;
1050         tasklet_schedule(&d40c->tasklet);
1051
1052 }
1053
1054 static void dma_tasklet(unsigned long data)
1055 {
1056         struct d40_chan *d40c = (struct d40_chan *) data;
1057         struct d40_desc *d40d;
1058         unsigned long flags;
1059         dma_async_tx_callback callback;
1060         void *callback_param;
1061
1062         spin_lock_irqsave(&d40c->lock, flags);
1063
1064         /* Get first active entry from list */
1065         d40d = d40_first_active_get(d40c);
1066
1067         if (d40d == NULL)
1068                 goto err;
1069
1070         d40c->completed = d40d->txd.cookie;
1071
1072         /*
1073          * If terminating a channel pending_tx is set to zero.
1074          * This prevents any finished active jobs to return to the client.
1075          */
1076         if (d40c->pending_tx == 0) {
1077                 spin_unlock_irqrestore(&d40c->lock, flags);
1078                 return;
1079         }
1080
1081         /* Callback to client */
1082         callback = d40d->txd.callback;
1083         callback_param = d40d->txd.callback_param;
1084
1085         if (async_tx_test_ack(&d40d->txd)) {
1086                 d40_pool_lli_free(d40c, d40d);
1087                 d40_desc_remove(d40d);
1088                 d40_desc_free(d40c, d40d);
1089         } else {
1090                 if (!d40d->is_in_client_list) {
1091                         d40_desc_remove(d40d);
1092                         d40_lcla_free_all(d40c, d40d);
1093                         list_add_tail(&d40d->node, &d40c->client);
1094                         d40d->is_in_client_list = true;
1095                 }
1096         }
1097
1098         d40c->pending_tx--;
1099
1100         if (d40c->pending_tx)
1101                 tasklet_schedule(&d40c->tasklet);
1102
1103         spin_unlock_irqrestore(&d40c->lock, flags);
1104
1105         if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1106                 callback(callback_param);
1107
1108         return;
1109
1110  err:
1111         /* Rescue manouver if receiving double interrupts */
1112         if (d40c->pending_tx > 0)
1113                 d40c->pending_tx--;
1114         spin_unlock_irqrestore(&d40c->lock, flags);
1115 }
1116
1117 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1118 {
1119         static const struct d40_interrupt_lookup il[] = {
1120                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
1121                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1122                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1123                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1124                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
1125                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
1126                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
1127                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
1128                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
1129                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
1130         };
1131
1132         int i;
1133         u32 regs[ARRAY_SIZE(il)];
1134         u32 idx;
1135         u32 row;
1136         long chan = -1;
1137         struct d40_chan *d40c;
1138         unsigned long flags;
1139         struct d40_base *base = data;
1140
1141         spin_lock_irqsave(&base->interrupt_lock, flags);
1142
1143         /* Read interrupt status of both logical and physical channels */
1144         for (i = 0; i < ARRAY_SIZE(il); i++)
1145                 regs[i] = readl(base->virtbase + il[i].src);
1146
1147         for (;;) {
1148
1149                 chan = find_next_bit((unsigned long *)regs,
1150                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1151
1152                 /* No more set bits found? */
1153                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1154                         break;
1155
1156                 row = chan / BITS_PER_LONG;
1157                 idx = chan & (BITS_PER_LONG - 1);
1158
1159                 /* ACK interrupt */
1160                 writel(1 << idx, base->virtbase + il[row].clr);
1161
1162                 if (il[row].offset == D40_PHY_CHAN)
1163                         d40c = base->lookup_phy_chans[idx];
1164                 else
1165                         d40c = base->lookup_log_chans[il[row].offset + idx];
1166                 spin_lock(&d40c->lock);
1167
1168                 if (!il[row].is_error)
1169                         dma_tc_handle(d40c);
1170                 else
1171                         d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1172                                 chan, il[row].offset, idx);
1173
1174                 spin_unlock(&d40c->lock);
1175         }
1176
1177         spin_unlock_irqrestore(&base->interrupt_lock, flags);
1178
1179         return IRQ_HANDLED;
1180 }
1181
1182 static int d40_validate_conf(struct d40_chan *d40c,
1183                              struct stedma40_chan_cfg *conf)
1184 {
1185         int res = 0;
1186         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1187         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1188         bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1189
1190         if (!conf->dir) {
1191                 chan_err(d40c, "Invalid direction.\n");
1192                 res = -EINVAL;
1193         }
1194
1195         if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1196             d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1197             d40c->runtime_addr == 0) {
1198
1199                 chan_err(d40c, "Invalid TX channel address (%d)\n",
1200                          conf->dst_dev_type);
1201                 res = -EINVAL;
1202         }
1203
1204         if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1205             d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1206             d40c->runtime_addr == 0) {
1207                 chan_err(d40c, "Invalid RX channel address (%d)\n",
1208                         conf->src_dev_type);
1209                 res = -EINVAL;
1210         }
1211
1212         if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1213             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1214                 chan_err(d40c, "Invalid dst\n");
1215                 res = -EINVAL;
1216         }
1217
1218         if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1219             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1220                 chan_err(d40c, "Invalid src\n");
1221                 res = -EINVAL;
1222         }
1223
1224         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1225             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1226                 chan_err(d40c, "No event line\n");
1227                 res = -EINVAL;
1228         }
1229
1230         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1231             (src_event_group != dst_event_group)) {
1232                 chan_err(d40c, "Invalid event group\n");
1233                 res = -EINVAL;
1234         }
1235
1236         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1237                 /*
1238                  * DMAC HW supports it. Will be added to this driver,
1239                  * in case any dma client requires it.
1240                  */
1241                 chan_err(d40c, "periph to periph not supported\n");
1242                 res = -EINVAL;
1243         }
1244
1245         if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1246             (1 << conf->src_info.data_width) !=
1247             d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1248             (1 << conf->dst_info.data_width)) {
1249                 /*
1250                  * The DMAC hardware only supports
1251                  * src (burst x width) == dst (burst x width)
1252                  */
1253
1254                 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1255                 res = -EINVAL;
1256         }
1257
1258         return res;
1259 }
1260
1261 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1262                                int log_event_line, bool is_log)
1263 {
1264         unsigned long flags;
1265         spin_lock_irqsave(&phy->lock, flags);
1266         if (!is_log) {
1267                 /* Physical interrupts are masked per physical full channel */
1268                 if (phy->allocated_src == D40_ALLOC_FREE &&
1269                     phy->allocated_dst == D40_ALLOC_FREE) {
1270                         phy->allocated_dst = D40_ALLOC_PHY;
1271                         phy->allocated_src = D40_ALLOC_PHY;
1272                         goto found;
1273                 } else
1274                         goto not_found;
1275         }
1276
1277         /* Logical channel */
1278         if (is_src) {
1279                 if (phy->allocated_src == D40_ALLOC_PHY)
1280                         goto not_found;
1281
1282                 if (phy->allocated_src == D40_ALLOC_FREE)
1283                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1284
1285                 if (!(phy->allocated_src & (1 << log_event_line))) {
1286                         phy->allocated_src |= 1 << log_event_line;
1287                         goto found;
1288                 } else
1289                         goto not_found;
1290         } else {
1291                 if (phy->allocated_dst == D40_ALLOC_PHY)
1292                         goto not_found;
1293
1294                 if (phy->allocated_dst == D40_ALLOC_FREE)
1295                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1296
1297                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1298                         phy->allocated_dst |= 1 << log_event_line;
1299                         goto found;
1300                 } else
1301                         goto not_found;
1302         }
1303
1304 not_found:
1305         spin_unlock_irqrestore(&phy->lock, flags);
1306         return false;
1307 found:
1308         spin_unlock_irqrestore(&phy->lock, flags);
1309         return true;
1310 }
1311
1312 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1313                                int log_event_line)
1314 {
1315         unsigned long flags;
1316         bool is_free = false;
1317
1318         spin_lock_irqsave(&phy->lock, flags);
1319         if (!log_event_line) {
1320                 phy->allocated_dst = D40_ALLOC_FREE;
1321                 phy->allocated_src = D40_ALLOC_FREE;
1322                 is_free = true;
1323                 goto out;
1324         }
1325
1326         /* Logical channel */
1327         if (is_src) {
1328                 phy->allocated_src &= ~(1 << log_event_line);
1329                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1330                         phy->allocated_src = D40_ALLOC_FREE;
1331         } else {
1332                 phy->allocated_dst &= ~(1 << log_event_line);
1333                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1334                         phy->allocated_dst = D40_ALLOC_FREE;
1335         }
1336
1337         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1338                    D40_ALLOC_FREE);
1339
1340 out:
1341         spin_unlock_irqrestore(&phy->lock, flags);
1342
1343         return is_free;
1344 }
1345
1346 static int d40_allocate_channel(struct d40_chan *d40c)
1347 {
1348         int dev_type;
1349         int event_group;
1350         int event_line;
1351         struct d40_phy_res *phys;
1352         int i;
1353         int j;
1354         int log_num;
1355         bool is_src;
1356         bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1357
1358         phys = d40c->base->phy_res;
1359
1360         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1361                 dev_type = d40c->dma_cfg.src_dev_type;
1362                 log_num = 2 * dev_type;
1363                 is_src = true;
1364         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1365                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1366                 /* dst event lines are used for logical memcpy */
1367                 dev_type = d40c->dma_cfg.dst_dev_type;
1368                 log_num = 2 * dev_type + 1;
1369                 is_src = false;
1370         } else
1371                 return -EINVAL;
1372
1373         event_group = D40_TYPE_TO_GROUP(dev_type);
1374         event_line = D40_TYPE_TO_EVENT(dev_type);
1375
1376         if (!is_log) {
1377                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1378                         /* Find physical half channel */
1379                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1380
1381                                 if (d40_alloc_mask_set(&phys[i], is_src,
1382                                                        0, is_log))
1383                                         goto found_phy;
1384                         }
1385                 } else
1386                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1387                                 int phy_num = j  + event_group * 2;
1388                                 for (i = phy_num; i < phy_num + 2; i++) {
1389                                         if (d40_alloc_mask_set(&phys[i],
1390                                                                is_src,
1391                                                                0,
1392                                                                is_log))
1393                                                 goto found_phy;
1394                                 }
1395                         }
1396                 return -EINVAL;
1397 found_phy:
1398                 d40c->phy_chan = &phys[i];
1399                 d40c->log_num = D40_PHY_CHAN;
1400                 goto out;
1401         }
1402         if (dev_type == -1)
1403                 return -EINVAL;
1404
1405         /* Find logical channel */
1406         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1407                 int phy_num = j + event_group * 2;
1408                 /*
1409                  * Spread logical channels across all available physical rather
1410                  * than pack every logical channel at the first available phy
1411                  * channels.
1412                  */
1413                 if (is_src) {
1414                         for (i = phy_num; i < phy_num + 2; i++) {
1415                                 if (d40_alloc_mask_set(&phys[i], is_src,
1416                                                        event_line, is_log))
1417                                         goto found_log;
1418                         }
1419                 } else {
1420                         for (i = phy_num + 1; i >= phy_num; i--) {
1421                                 if (d40_alloc_mask_set(&phys[i], is_src,
1422                                                        event_line, is_log))
1423                                         goto found_log;
1424                         }
1425                 }
1426         }
1427         return -EINVAL;
1428
1429 found_log:
1430         d40c->phy_chan = &phys[i];
1431         d40c->log_num = log_num;
1432 out:
1433
1434         if (is_log)
1435                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1436         else
1437                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1438
1439         return 0;
1440
1441 }
1442
1443 static int d40_config_memcpy(struct d40_chan *d40c)
1444 {
1445         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1446
1447         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1448                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1449                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1450                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1451                         memcpy[d40c->chan.chan_id];
1452
1453         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1454                    dma_has_cap(DMA_SLAVE, cap)) {
1455                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1456         } else {
1457                 chan_err(d40c, "No memcpy\n");
1458                 return -EINVAL;
1459         }
1460
1461         return 0;
1462 }
1463
1464
1465 static int d40_free_dma(struct d40_chan *d40c)
1466 {
1467
1468         int res = 0;
1469         u32 event;
1470         struct d40_phy_res *phy = d40c->phy_chan;
1471         bool is_src;
1472         struct d40_desc *d;
1473         struct d40_desc *_d;
1474
1475
1476         /* Terminate all queued and active transfers */
1477         d40_term_all(d40c);
1478
1479         /* Release client owned descriptors */
1480         if (!list_empty(&d40c->client))
1481                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1482                         d40_pool_lli_free(d40c, d);
1483                         d40_desc_remove(d);
1484                         d40_desc_free(d40c, d);
1485                 }
1486
1487         if (phy == NULL) {
1488                 chan_err(d40c, "phy == null\n");
1489                 return -EINVAL;
1490         }
1491
1492         if (phy->allocated_src == D40_ALLOC_FREE &&
1493             phy->allocated_dst == D40_ALLOC_FREE) {
1494                 chan_err(d40c, "channel already free\n");
1495                 return -EINVAL;
1496         }
1497
1498         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1499             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1500                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1501                 is_src = false;
1502         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1503                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1504                 is_src = true;
1505         } else {
1506                 chan_err(d40c, "Unknown direction\n");
1507                 return -EINVAL;
1508         }
1509
1510         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1511         if (res) {
1512                 chan_err(d40c, "suspend failed\n");
1513                 return res;
1514         }
1515
1516         if (chan_is_logical(d40c)) {
1517                 /* Release logical channel, deactivate the event line */
1518
1519                 d40_config_set_event(d40c, false);
1520                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1521
1522                 /*
1523                  * Check if there are more logical allocation
1524                  * on this phy channel.
1525                  */
1526                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1527                         /* Resume the other logical channels if any */
1528                         if (d40_chan_has_events(d40c)) {
1529                                 res = d40_channel_execute_command(d40c,
1530                                                                   D40_DMA_RUN);
1531                                 if (res) {
1532                                         chan_err(d40c,
1533                                                 "Executing RUN command\n");
1534                                         return res;
1535                                 }
1536                         }
1537                         return 0;
1538                 }
1539         } else {
1540                 (void) d40_alloc_mask_free(phy, is_src, 0);
1541         }
1542
1543         /* Release physical channel */
1544         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1545         if (res) {
1546                 chan_err(d40c, "Failed to stop channel\n");
1547                 return res;
1548         }
1549         d40c->phy_chan = NULL;
1550         d40c->configured = false;
1551         d40c->base->lookup_phy_chans[phy->num] = NULL;
1552
1553         return 0;
1554 }
1555
1556 static bool d40_is_paused(struct d40_chan *d40c)
1557 {
1558         void __iomem *chanbase = chan_base(d40c);
1559         bool is_paused = false;
1560         unsigned long flags;
1561         void __iomem *active_reg;
1562         u32 status;
1563         u32 event;
1564
1565         spin_lock_irqsave(&d40c->lock, flags);
1566
1567         if (chan_is_physical(d40c)) {
1568                 if (d40c->phy_chan->num % 2 == 0)
1569                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1570                 else
1571                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1572
1573                 status = (readl(active_reg) &
1574                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1575                         D40_CHAN_POS(d40c->phy_chan->num);
1576                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1577                         is_paused = true;
1578
1579                 goto _exit;
1580         }
1581
1582         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1583             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1584                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1585                 status = readl(chanbase + D40_CHAN_REG_SDLNK);
1586         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1587                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1588                 status = readl(chanbase + D40_CHAN_REG_SSLNK);
1589         } else {
1590                 chan_err(d40c, "Unknown direction\n");
1591                 goto _exit;
1592         }
1593
1594         status = (status & D40_EVENTLINE_MASK(event)) >>
1595                 D40_EVENTLINE_POS(event);
1596
1597         if (status != D40_DMA_RUN)
1598                 is_paused = true;
1599 _exit:
1600         spin_unlock_irqrestore(&d40c->lock, flags);
1601         return is_paused;
1602
1603 }
1604
1605
1606 static u32 stedma40_residue(struct dma_chan *chan)
1607 {
1608         struct d40_chan *d40c =
1609                 container_of(chan, struct d40_chan, chan);
1610         u32 bytes_left;
1611         unsigned long flags;
1612
1613         spin_lock_irqsave(&d40c->lock, flags);
1614         bytes_left = d40_residue(d40c);
1615         spin_unlock_irqrestore(&d40c->lock, flags);
1616
1617         return bytes_left;
1618 }
1619
1620 static struct d40_desc *
1621 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1622               unsigned int sg_len, unsigned long dma_flags)
1623 {
1624         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1625         struct d40_desc *desc;
1626
1627         desc = d40_desc_get(chan);
1628         if (!desc)
1629                 return NULL;
1630
1631         desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1632                                         cfg->dst_info.data_width);
1633         if (desc->lli_len < 0) {
1634                 chan_err(chan, "Unaligned size\n");
1635                 d40_desc_free(chan, desc);
1636
1637                 return NULL;
1638         }
1639
1640         desc->lli_current = 0;
1641         desc->txd.flags = dma_flags;
1642         desc->txd.tx_submit = d40_tx_submit;
1643
1644         dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1645
1646         return desc;
1647 }
1648
1649 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1650                                                    struct scatterlist *sgl_dst,
1651                                                    struct scatterlist *sgl_src,
1652                                                    unsigned int sgl_len,
1653                                                    unsigned long dma_flags)
1654 {
1655         int res;
1656         struct d40_desc *d40d;
1657         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1658                                              chan);
1659         unsigned long flags;
1660
1661         if (d40c->phy_chan == NULL) {
1662                 chan_err(d40c, "Unallocated channel.\n");
1663                 return ERR_PTR(-EINVAL);
1664         }
1665
1666         spin_lock_irqsave(&d40c->lock, flags);
1667
1668         d40d = d40_prep_desc(d40c, sgl_dst, sgl_len, dma_flags);
1669         if (!d40d)
1670                 goto err;
1671
1672         if (chan_is_logical(d40c)) {
1673
1674                 if (d40_pool_lli_alloc(d40c, d40d, d40d->lli_len, true) < 0) {
1675                         chan_err(d40c, "Out of memory\n");
1676                         goto err;
1677                 }
1678
1679                 (void) d40_log_sg_to_lli(sgl_src,
1680                                          sgl_len,
1681                                          d40d->lli_log.src,
1682                                          d40c->log_def.lcsp1,
1683                                          d40c->dma_cfg.src_info.data_width,
1684                                          d40c->dma_cfg.dst_info.data_width);
1685
1686                 (void) d40_log_sg_to_lli(sgl_dst,
1687                                          sgl_len,
1688                                          d40d->lli_log.dst,
1689                                          d40c->log_def.lcsp3,
1690                                          d40c->dma_cfg.dst_info.data_width,
1691                                          d40c->dma_cfg.src_info.data_width);
1692         } else {
1693                 if (d40_pool_lli_alloc(d40c, d40d, d40d->lli_len, false) < 0) {
1694                         chan_err(d40c, "Out of memory\n");
1695                         goto err;
1696                 }
1697
1698                 res = d40_phy_sg_to_lli(sgl_src,
1699                                         sgl_len,
1700                                         0,
1701                                         d40d->lli_phy.src,
1702                                         virt_to_phys(d40d->lli_phy.src),
1703                                         d40c->src_def_cfg,
1704                                         d40c->dma_cfg.src_info.data_width,
1705                                         d40c->dma_cfg.dst_info.data_width,
1706                                         d40c->dma_cfg.src_info.psize);
1707
1708                 if (res < 0)
1709                         goto err;
1710
1711                 res = d40_phy_sg_to_lli(sgl_dst,
1712                                         sgl_len,
1713                                         0,
1714                                         d40d->lli_phy.dst,
1715                                         virt_to_phys(d40d->lli_phy.dst),
1716                                         d40c->dst_def_cfg,
1717                                         d40c->dma_cfg.dst_info.data_width,
1718                                         d40c->dma_cfg.src_info.data_width,
1719                                         d40c->dma_cfg.dst_info.psize);
1720
1721                 if (res < 0)
1722                         goto err;
1723
1724                 dma_sync_single_for_device(d40c->base->dev,
1725                                            d40d->lli_pool.dma_addr,
1726                                            d40d->lli_pool.size, DMA_TO_DEVICE);
1727         }
1728
1729         spin_unlock_irqrestore(&d40c->lock, flags);
1730
1731         return &d40d->txd;
1732 err:
1733         if (d40d)
1734                 d40_desc_free(d40c, d40d);
1735         spin_unlock_irqrestore(&d40c->lock, flags);
1736         return NULL;
1737 }
1738 EXPORT_SYMBOL(stedma40_memcpy_sg);
1739
1740 bool stedma40_filter(struct dma_chan *chan, void *data)
1741 {
1742         struct stedma40_chan_cfg *info = data;
1743         struct d40_chan *d40c =
1744                 container_of(chan, struct d40_chan, chan);
1745         int err;
1746
1747         if (data) {
1748                 err = d40_validate_conf(d40c, info);
1749                 if (!err)
1750                         d40c->dma_cfg = *info;
1751         } else
1752                 err = d40_config_memcpy(d40c);
1753
1754         if (!err)
1755                 d40c->configured = true;
1756
1757         return err == 0;
1758 }
1759 EXPORT_SYMBOL(stedma40_filter);
1760
1761 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1762 {
1763         bool realtime = d40c->dma_cfg.realtime;
1764         bool highprio = d40c->dma_cfg.high_priority;
1765         u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1766         u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1767         u32 event = D40_TYPE_TO_EVENT(dev_type);
1768         u32 group = D40_TYPE_TO_GROUP(dev_type);
1769         u32 bit = 1 << event;
1770
1771         /* Destination event lines are stored in the upper halfword */
1772         if (!src)
1773                 bit <<= 16;
1774
1775         writel(bit, d40c->base->virtbase + prioreg + group * 4);
1776         writel(bit, d40c->base->virtbase + rtreg + group * 4);
1777 }
1778
1779 static void d40_set_prio_realtime(struct d40_chan *d40c)
1780 {
1781         if (d40c->base->rev < 3)
1782                 return;
1783
1784         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
1785             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1786                 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1787
1788         if ((d40c->dma_cfg.dir ==  STEDMA40_MEM_TO_PERIPH) ||
1789             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1790                 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1791 }
1792
1793 /* DMA ENGINE functions */
1794 static int d40_alloc_chan_resources(struct dma_chan *chan)
1795 {
1796         int err;
1797         unsigned long flags;
1798         struct d40_chan *d40c =
1799                 container_of(chan, struct d40_chan, chan);
1800         bool is_free_phy;
1801         spin_lock_irqsave(&d40c->lock, flags);
1802
1803         d40c->completed = chan->cookie = 1;
1804
1805         /* If no dma configuration is set use default configuration (memcpy) */
1806         if (!d40c->configured) {
1807                 err = d40_config_memcpy(d40c);
1808                 if (err) {
1809                         chan_err(d40c, "Failed to configure memcpy channel\n");
1810                         goto fail;
1811                 }
1812         }
1813         is_free_phy = (d40c->phy_chan == NULL);
1814
1815         err = d40_allocate_channel(d40c);
1816         if (err) {
1817                 chan_err(d40c, "Failed to allocate channel\n");
1818                 goto fail;
1819         }
1820
1821         /* Fill in basic CFG register values */
1822         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1823                     &d40c->dst_def_cfg, chan_is_logical(d40c));
1824
1825         d40_set_prio_realtime(d40c);
1826
1827         if (chan_is_logical(d40c)) {
1828                 d40_log_cfg(&d40c->dma_cfg,
1829                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1830
1831                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1832                         d40c->lcpa = d40c->base->lcpa_base +
1833                           d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1834                 else
1835                         d40c->lcpa = d40c->base->lcpa_base +
1836                           d40c->dma_cfg.dst_dev_type *
1837                           D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1838         }
1839
1840         /*
1841          * Only write channel configuration to the DMA if the physical
1842          * resource is free. In case of multiple logical channels
1843          * on the same physical resource, only the first write is necessary.
1844          */
1845         if (is_free_phy)
1846                 d40_config_write(d40c);
1847 fail:
1848         spin_unlock_irqrestore(&d40c->lock, flags);
1849         return err;
1850 }
1851
1852 static void d40_free_chan_resources(struct dma_chan *chan)
1853 {
1854         struct d40_chan *d40c =
1855                 container_of(chan, struct d40_chan, chan);
1856         int err;
1857         unsigned long flags;
1858
1859         if (d40c->phy_chan == NULL) {
1860                 chan_err(d40c, "Cannot free unallocated channel\n");
1861                 return;
1862         }
1863
1864
1865         spin_lock_irqsave(&d40c->lock, flags);
1866
1867         err = d40_free_dma(d40c);
1868
1869         if (err)
1870                 chan_err(d40c, "Failed to free channel\n");
1871         spin_unlock_irqrestore(&d40c->lock, flags);
1872 }
1873
1874 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1875                                                        dma_addr_t dst,
1876                                                        dma_addr_t src,
1877                                                        size_t size,
1878                                                        unsigned long dma_flags)
1879 {
1880         struct scatterlist dst_sg;
1881         struct scatterlist src_sg;
1882
1883         sg_init_table(&dst_sg, 1);
1884         sg_init_table(&src_sg, 1);
1885
1886         sg_dma_address(&dst_sg) = dst;
1887         sg_dma_address(&src_sg) = src;
1888
1889         sg_dma_len(&dst_sg) = size;
1890         sg_dma_len(&src_sg) = size;
1891
1892         return stedma40_memcpy_sg(chan, &dst_sg, &src_sg, 1, dma_flags);
1893 }
1894
1895 static struct dma_async_tx_descriptor *
1896 d40_prep_sg(struct dma_chan *chan,
1897             struct scatterlist *dst_sg, unsigned int dst_nents,
1898             struct scatterlist *src_sg, unsigned int src_nents,
1899             unsigned long dma_flags)
1900 {
1901         if (dst_nents != src_nents)
1902                 return NULL;
1903
1904         return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
1905 }
1906
1907 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1908                                  struct d40_chan *d40c,
1909                                  struct scatterlist *sgl,
1910                                  unsigned int sg_len,
1911                                  enum dma_data_direction direction,
1912                                  unsigned long dma_flags)
1913 {
1914         dma_addr_t dev_addr = 0;
1915         int total_size;
1916
1917         if (d40_pool_lli_alloc(d40c, d40d, d40d->lli_len, true) < 0) {
1918                 chan_err(d40c, "Out of memory\n");
1919                 return -ENOMEM;
1920         }
1921
1922         if (direction == DMA_FROM_DEVICE)
1923                 if (d40c->runtime_addr)
1924                         dev_addr = d40c->runtime_addr;
1925                 else
1926                         dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1927         else if (direction == DMA_TO_DEVICE)
1928                 if (d40c->runtime_addr)
1929                         dev_addr = d40c->runtime_addr;
1930                 else
1931                         dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1932
1933         else
1934                 return -EINVAL;
1935
1936         total_size = d40_log_sg_to_dev(sgl, sg_len,
1937                                        &d40d->lli_log,
1938                                        &d40c->log_def,
1939                                        d40c->dma_cfg.src_info.data_width,
1940                                        d40c->dma_cfg.dst_info.data_width,
1941                                        direction,
1942                                        dev_addr);
1943
1944         if (total_size < 0)
1945                 return -EINVAL;
1946
1947         return 0;
1948 }
1949
1950 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1951                                  struct d40_chan *d40c,
1952                                  struct scatterlist *sgl,
1953                                  unsigned int sgl_len,
1954                                  enum dma_data_direction direction,
1955                                  unsigned long dma_flags)
1956 {
1957         dma_addr_t src_dev_addr;
1958         dma_addr_t dst_dev_addr;
1959         int res;
1960
1961         if (d40_pool_lli_alloc(d40c, d40d, d40d->lli_len, false) < 0) {
1962                 chan_err(d40c, "Out of memory\n");
1963                 return -ENOMEM;
1964         }
1965
1966         if (direction == DMA_FROM_DEVICE) {
1967                 dst_dev_addr = 0;
1968                 if (d40c->runtime_addr)
1969                         src_dev_addr = d40c->runtime_addr;
1970                 else
1971                         src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1972         } else if (direction == DMA_TO_DEVICE) {
1973                 if (d40c->runtime_addr)
1974                         dst_dev_addr = d40c->runtime_addr;
1975                 else
1976                         dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1977                 src_dev_addr = 0;
1978         } else
1979                 return -EINVAL;
1980
1981         res = d40_phy_sg_to_lli(sgl,
1982                                 sgl_len,
1983                                 src_dev_addr,
1984                                 d40d->lli_phy.src,
1985                                 virt_to_phys(d40d->lli_phy.src),
1986                                 d40c->src_def_cfg,
1987                                 d40c->dma_cfg.src_info.data_width,
1988                                 d40c->dma_cfg.dst_info.data_width,
1989                                 d40c->dma_cfg.src_info.psize);
1990         if (res < 0)
1991                 return res;
1992
1993         res = d40_phy_sg_to_lli(sgl,
1994                                 sgl_len,
1995                                 dst_dev_addr,
1996                                 d40d->lli_phy.dst,
1997                                 virt_to_phys(d40d->lli_phy.dst),
1998                                 d40c->dst_def_cfg,
1999                                 d40c->dma_cfg.dst_info.data_width,
2000                                 d40c->dma_cfg.src_info.data_width,
2001                                 d40c->dma_cfg.dst_info.psize);
2002         if (res < 0)
2003                 return res;
2004
2005         dma_sync_single_for_device(d40c->base->dev, d40d->lli_pool.dma_addr,
2006                                    d40d->lli_pool.size, DMA_TO_DEVICE);
2007         return 0;
2008 }
2009
2010 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2011                                                          struct scatterlist *sgl,
2012                                                          unsigned int sg_len,
2013                                                          enum dma_data_direction direction,
2014                                                          unsigned long dma_flags)
2015 {
2016         struct d40_desc *d40d;
2017         struct d40_chan *d40c = container_of(chan, struct d40_chan,
2018                                              chan);
2019         unsigned long flags;
2020         int err;
2021
2022         if (d40c->phy_chan == NULL) {
2023                 chan_err(d40c, "Cannot prepare unallocated channel\n");
2024                 return ERR_PTR(-EINVAL);
2025         }
2026
2027         spin_lock_irqsave(&d40c->lock, flags);
2028
2029         d40d = d40_prep_desc(d40c, sgl, sg_len, dma_flags);
2030         if (d40d == NULL)
2031                 goto err;
2032
2033         if (chan_is_logical(d40c))
2034                 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2035                                             direction, dma_flags);
2036         else
2037                 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2038                                             direction, dma_flags);
2039         if (err) {
2040                 chan_err(d40c, "Failed to prepare %s slave sg job: %d\n",
2041                         chan_is_logical(d40c) ? "log" : "phy", err);
2042                 goto err;
2043         }
2044
2045         spin_unlock_irqrestore(&d40c->lock, flags);
2046         return &d40d->txd;
2047
2048 err:
2049         if (d40d)
2050                 d40_desc_free(d40c, d40d);
2051         spin_unlock_irqrestore(&d40c->lock, flags);
2052         return NULL;
2053 }
2054
2055 static enum dma_status d40_tx_status(struct dma_chan *chan,
2056                                      dma_cookie_t cookie,
2057                                      struct dma_tx_state *txstate)
2058 {
2059         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2060         dma_cookie_t last_used;
2061         dma_cookie_t last_complete;
2062         int ret;
2063
2064         if (d40c->phy_chan == NULL) {
2065                 chan_err(d40c, "Cannot read status of unallocated channel\n");
2066                 return -EINVAL;
2067         }
2068
2069         last_complete = d40c->completed;
2070         last_used = chan->cookie;
2071
2072         if (d40_is_paused(d40c))
2073                 ret = DMA_PAUSED;
2074         else
2075                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2076
2077         dma_set_tx_state(txstate, last_complete, last_used,
2078                          stedma40_residue(chan));
2079
2080         return ret;
2081 }
2082
2083 static void d40_issue_pending(struct dma_chan *chan)
2084 {
2085         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2086         unsigned long flags;
2087
2088         if (d40c->phy_chan == NULL) {
2089                 chan_err(d40c, "Channel is not allocated!\n");
2090                 return;
2091         }
2092
2093         spin_lock_irqsave(&d40c->lock, flags);
2094
2095         /* Busy means that pending jobs are already being processed */
2096         if (!d40c->busy)
2097                 (void) d40_queue_start(d40c);
2098
2099         spin_unlock_irqrestore(&d40c->lock, flags);
2100 }
2101
2102 /* Runtime reconfiguration extension */
2103 static void d40_set_runtime_config(struct dma_chan *chan,
2104                                struct dma_slave_config *config)
2105 {
2106         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2107         struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2108         enum dma_slave_buswidth config_addr_width;
2109         dma_addr_t config_addr;
2110         u32 config_maxburst;
2111         enum stedma40_periph_data_width addr_width;
2112         int psize;
2113
2114         if (config->direction == DMA_FROM_DEVICE) {
2115                 dma_addr_t dev_addr_rx =
2116                         d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2117
2118                 config_addr = config->src_addr;
2119                 if (dev_addr_rx)
2120                         dev_dbg(d40c->base->dev,
2121                                 "channel has a pre-wired RX address %08x "
2122                                 "overriding with %08x\n",
2123                                 dev_addr_rx, config_addr);
2124                 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2125                         dev_dbg(d40c->base->dev,
2126                                 "channel was not configured for peripheral "
2127                                 "to memory transfer (%d) overriding\n",
2128                                 cfg->dir);
2129                 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2130
2131                 config_addr_width = config->src_addr_width;
2132                 config_maxburst = config->src_maxburst;
2133
2134         } else if (config->direction == DMA_TO_DEVICE) {
2135                 dma_addr_t dev_addr_tx =
2136                         d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2137
2138                 config_addr = config->dst_addr;
2139                 if (dev_addr_tx)
2140                         dev_dbg(d40c->base->dev,
2141                                 "channel has a pre-wired TX address %08x "
2142                                 "overriding with %08x\n",
2143                                 dev_addr_tx, config_addr);
2144                 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2145                         dev_dbg(d40c->base->dev,
2146                                 "channel was not configured for memory "
2147                                 "to peripheral transfer (%d) overriding\n",
2148                                 cfg->dir);
2149                 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2150
2151                 config_addr_width = config->dst_addr_width;
2152                 config_maxburst = config->dst_maxburst;
2153
2154         } else {
2155                 dev_err(d40c->base->dev,
2156                         "unrecognized channel direction %d\n",
2157                         config->direction);
2158                 return;
2159         }
2160
2161         switch (config_addr_width) {
2162         case DMA_SLAVE_BUSWIDTH_1_BYTE:
2163                 addr_width = STEDMA40_BYTE_WIDTH;
2164                 break;
2165         case DMA_SLAVE_BUSWIDTH_2_BYTES:
2166                 addr_width = STEDMA40_HALFWORD_WIDTH;
2167                 break;
2168         case DMA_SLAVE_BUSWIDTH_4_BYTES:
2169                 addr_width = STEDMA40_WORD_WIDTH;
2170                 break;
2171         case DMA_SLAVE_BUSWIDTH_8_BYTES:
2172                 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2173                 break;
2174         default:
2175                 dev_err(d40c->base->dev,
2176                         "illegal peripheral address width "
2177                         "requested (%d)\n",
2178                         config->src_addr_width);
2179                 return;
2180         }
2181
2182         if (chan_is_logical(d40c)) {
2183                 if (config_maxburst >= 16)
2184                         psize = STEDMA40_PSIZE_LOG_16;
2185                 else if (config_maxburst >= 8)
2186                         psize = STEDMA40_PSIZE_LOG_8;
2187                 else if (config_maxburst >= 4)
2188                         psize = STEDMA40_PSIZE_LOG_4;
2189                 else
2190                         psize = STEDMA40_PSIZE_LOG_1;
2191         } else {
2192                 if (config_maxburst >= 16)
2193                         psize = STEDMA40_PSIZE_PHY_16;
2194                 else if (config_maxburst >= 8)
2195                         psize = STEDMA40_PSIZE_PHY_8;
2196                 else if (config_maxburst >= 4)
2197                         psize = STEDMA40_PSIZE_PHY_4;
2198                 else if (config_maxburst >= 2)
2199                         psize = STEDMA40_PSIZE_PHY_2;
2200                 else
2201                         psize = STEDMA40_PSIZE_PHY_1;
2202         }
2203
2204         /* Set up all the endpoint configs */
2205         cfg->src_info.data_width = addr_width;
2206         cfg->src_info.psize = psize;
2207         cfg->src_info.big_endian = false;
2208         cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2209         cfg->dst_info.data_width = addr_width;
2210         cfg->dst_info.psize = psize;
2211         cfg->dst_info.big_endian = false;
2212         cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2213
2214         /* Fill in register values */
2215         if (chan_is_logical(d40c))
2216                 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2217         else
2218                 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2219                             &d40c->dst_def_cfg, false);
2220
2221         /* These settings will take precedence later */
2222         d40c->runtime_addr = config_addr;
2223         d40c->runtime_direction = config->direction;
2224         dev_dbg(d40c->base->dev,
2225                 "configured channel %s for %s, data width %d, "
2226                 "maxburst %d bytes, LE, no flow control\n",
2227                 dma_chan_name(chan),
2228                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2229                 config_addr_width,
2230                 config_maxburst);
2231 }
2232
2233 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2234                        unsigned long arg)
2235 {
2236         unsigned long flags;
2237         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2238
2239         if (d40c->phy_chan == NULL) {
2240                 chan_err(d40c, "Channel is not allocated!\n");
2241                 return -EINVAL;
2242         }
2243
2244         switch (cmd) {
2245         case DMA_TERMINATE_ALL:
2246                 spin_lock_irqsave(&d40c->lock, flags);
2247                 d40_term_all(d40c);
2248                 spin_unlock_irqrestore(&d40c->lock, flags);
2249                 return 0;
2250         case DMA_PAUSE:
2251                 return d40_pause(chan);
2252         case DMA_RESUME:
2253                 return d40_resume(chan);
2254         case DMA_SLAVE_CONFIG:
2255                 d40_set_runtime_config(chan,
2256                         (struct dma_slave_config *) arg);
2257                 return 0;
2258         default:
2259                 break;
2260         }
2261
2262         /* Other commands are unimplemented */
2263         return -ENXIO;
2264 }
2265
2266 /* Initialization functions */
2267
2268 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2269                                  struct d40_chan *chans, int offset,
2270                                  int num_chans)
2271 {
2272         int i = 0;
2273         struct d40_chan *d40c;
2274
2275         INIT_LIST_HEAD(&dma->channels);
2276
2277         for (i = offset; i < offset + num_chans; i++) {
2278                 d40c = &chans[i];
2279                 d40c->base = base;
2280                 d40c->chan.device = dma;
2281
2282                 spin_lock_init(&d40c->lock);
2283
2284                 d40c->log_num = D40_PHY_CHAN;
2285
2286                 INIT_LIST_HEAD(&d40c->active);
2287                 INIT_LIST_HEAD(&d40c->queue);
2288                 INIT_LIST_HEAD(&d40c->client);
2289
2290                 tasklet_init(&d40c->tasklet, dma_tasklet,
2291                              (unsigned long) d40c);
2292
2293                 list_add_tail(&d40c->chan.device_node,
2294                               &dma->channels);
2295         }
2296 }
2297
2298 static int __init d40_dmaengine_init(struct d40_base *base,
2299                                      int num_reserved_chans)
2300 {
2301         int err ;
2302
2303         d40_chan_init(base, &base->dma_slave, base->log_chans,
2304                       0, base->num_log_chans);
2305
2306         dma_cap_zero(base->dma_slave.cap_mask);
2307         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2308
2309         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2310         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2311         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2312         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2313         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2314         base->dma_slave.device_tx_status = d40_tx_status;
2315         base->dma_slave.device_issue_pending = d40_issue_pending;
2316         base->dma_slave.device_control = d40_control;
2317         base->dma_slave.dev = base->dev;
2318
2319         err = dma_async_device_register(&base->dma_slave);
2320
2321         if (err) {
2322                 d40_err(base->dev, "Failed to register slave channels\n");
2323                 goto failure1;
2324         }
2325
2326         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2327                       base->num_log_chans, base->plat_data->memcpy_len);
2328
2329         dma_cap_zero(base->dma_memcpy.cap_mask);
2330         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2331         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2332
2333         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2334         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2335         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2336         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2337         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2338         base->dma_memcpy.device_tx_status = d40_tx_status;
2339         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2340         base->dma_memcpy.device_control = d40_control;
2341         base->dma_memcpy.dev = base->dev;
2342         /*
2343          * This controller can only access address at even
2344          * 32bit boundaries, i.e. 2^2
2345          */
2346         base->dma_memcpy.copy_align = 2;
2347
2348         err = dma_async_device_register(&base->dma_memcpy);
2349
2350         if (err) {
2351                 d40_err(base->dev,
2352                         "Failed to regsiter memcpy only channels\n");
2353                 goto failure2;
2354         }
2355
2356         d40_chan_init(base, &base->dma_both, base->phy_chans,
2357                       0, num_reserved_chans);
2358
2359         dma_cap_zero(base->dma_both.cap_mask);
2360         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2361         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2362         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2363
2364         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2365         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2366         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2367         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2368         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2369         base->dma_both.device_tx_status = d40_tx_status;
2370         base->dma_both.device_issue_pending = d40_issue_pending;
2371         base->dma_both.device_control = d40_control;
2372         base->dma_both.dev = base->dev;
2373         base->dma_both.copy_align = 2;
2374         err = dma_async_device_register(&base->dma_both);
2375
2376         if (err) {
2377                 d40_err(base->dev,
2378                         "Failed to register logical and physical capable channels\n");
2379                 goto failure3;
2380         }
2381         return 0;
2382 failure3:
2383         dma_async_device_unregister(&base->dma_memcpy);
2384 failure2:
2385         dma_async_device_unregister(&base->dma_slave);
2386 failure1:
2387         return err;
2388 }
2389
2390 /* Initialization functions. */
2391
2392 static int __init d40_phy_res_init(struct d40_base *base)
2393 {
2394         int i;
2395         int num_phy_chans_avail = 0;
2396         u32 val[2];
2397         int odd_even_bit = -2;
2398
2399         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2400         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2401
2402         for (i = 0; i < base->num_phy_chans; i++) {
2403                 base->phy_res[i].num = i;
2404                 odd_even_bit += 2 * ((i % 2) == 0);
2405                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2406                         /* Mark security only channels as occupied */
2407                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2408                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2409                 } else {
2410                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2411                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2412                         num_phy_chans_avail++;
2413                 }
2414                 spin_lock_init(&base->phy_res[i].lock);
2415         }
2416
2417         /* Mark disabled channels as occupied */
2418         for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2419                 int chan = base->plat_data->disabled_channels[i];
2420
2421                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2422                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2423                 num_phy_chans_avail--;
2424         }
2425
2426         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2427                  num_phy_chans_avail, base->num_phy_chans);
2428
2429         /* Verify settings extended vs standard */
2430         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2431
2432         for (i = 0; i < base->num_phy_chans; i++) {
2433
2434                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2435                     (val[0] & 0x3) != 1)
2436                         dev_info(base->dev,
2437                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2438                                  __func__, i, val[0] & 0x3);
2439
2440                 val[0] = val[0] >> 2;
2441         }
2442
2443         return num_phy_chans_avail;
2444 }
2445
2446 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2447 {
2448         static const struct d40_reg_val dma_id_regs[] = {
2449                 /* Peripheral Id */
2450                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2451                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2452                 /*
2453                  * D40_DREG_PERIPHID2 Depends on HW revision:
2454                  *  DB8500ed has 0x0008,
2455                  *  ? has 0x0018,
2456                  *  DB8500v1 has 0x0028
2457                  *  DB8500v2 has 0x0038
2458                  */
2459                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2460
2461                 /* PCell Id */
2462                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2463                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2464                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2465                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2466         };
2467         struct stedma40_platform_data *plat_data;
2468         struct clk *clk = NULL;
2469         void __iomem *virtbase = NULL;
2470         struct resource *res = NULL;
2471         struct d40_base *base = NULL;
2472         int num_log_chans = 0;
2473         int num_phy_chans;
2474         int i;
2475         u32 val;
2476         u32 rev;
2477
2478         clk = clk_get(&pdev->dev, NULL);
2479
2480         if (IS_ERR(clk)) {
2481                 d40_err(&pdev->dev, "No matching clock found\n");
2482                 goto failure;
2483         }
2484
2485         clk_enable(clk);
2486
2487         /* Get IO for DMAC base address */
2488         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2489         if (!res)
2490                 goto failure;
2491
2492         if (request_mem_region(res->start, resource_size(res),
2493                                D40_NAME " I/O base") == NULL)
2494                 goto failure;
2495
2496         virtbase = ioremap(res->start, resource_size(res));
2497         if (!virtbase)
2498                 goto failure;
2499
2500         /* HW version check */
2501         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2502                 if (dma_id_regs[i].val !=
2503                     readl(virtbase + dma_id_regs[i].reg)) {
2504                         d40_err(&pdev->dev,
2505                                 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2506                                 dma_id_regs[i].val,
2507                                 dma_id_regs[i].reg,
2508                                 readl(virtbase + dma_id_regs[i].reg));
2509                         goto failure;
2510                 }
2511         }
2512
2513         /* Get silicon revision and designer */
2514         val = readl(virtbase + D40_DREG_PERIPHID2);
2515
2516         if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2517             D40_HW_DESIGNER) {
2518                 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2519                         val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2520                         D40_HW_DESIGNER);
2521                 goto failure;
2522         }
2523
2524         rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2525                 D40_DREG_PERIPHID2_REV_POS;
2526
2527         /* The number of physical channels on this HW */
2528         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2529
2530         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2531                  rev, res->start);
2532
2533         plat_data = pdev->dev.platform_data;
2534
2535         /* Count the number of logical channels in use */
2536         for (i = 0; i < plat_data->dev_len; i++)
2537                 if (plat_data->dev_rx[i] != 0)
2538                         num_log_chans++;
2539
2540         for (i = 0; i < plat_data->dev_len; i++)
2541                 if (plat_data->dev_tx[i] != 0)
2542                         num_log_chans++;
2543
2544         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2545                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2546                        sizeof(struct d40_chan), GFP_KERNEL);
2547
2548         if (base == NULL) {
2549                 d40_err(&pdev->dev, "Out of memory\n");
2550                 goto failure;
2551         }
2552
2553         base->rev = rev;
2554         base->clk = clk;
2555         base->num_phy_chans = num_phy_chans;
2556         base->num_log_chans = num_log_chans;
2557         base->phy_start = res->start;
2558         base->phy_size = resource_size(res);
2559         base->virtbase = virtbase;
2560         base->plat_data = plat_data;
2561         base->dev = &pdev->dev;
2562         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2563         base->log_chans = &base->phy_chans[num_phy_chans];
2564
2565         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2566                                 GFP_KERNEL);
2567         if (!base->phy_res)
2568                 goto failure;
2569
2570         base->lookup_phy_chans = kzalloc(num_phy_chans *
2571                                          sizeof(struct d40_chan *),
2572                                          GFP_KERNEL);
2573         if (!base->lookup_phy_chans)
2574                 goto failure;
2575
2576         if (num_log_chans + plat_data->memcpy_len) {
2577                 /*
2578                  * The max number of logical channels are event lines for all
2579                  * src devices and dst devices
2580                  */
2581                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2582                                                  sizeof(struct d40_chan *),
2583                                                  GFP_KERNEL);
2584                 if (!base->lookup_log_chans)
2585                         goto failure;
2586         }
2587
2588         base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2589                                             sizeof(struct d40_desc *) *
2590                                             D40_LCLA_LINK_PER_EVENT_GRP,
2591                                             GFP_KERNEL);
2592         if (!base->lcla_pool.alloc_map)
2593                 goto failure;
2594
2595         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2596                                             0, SLAB_HWCACHE_ALIGN,
2597                                             NULL);
2598         if (base->desc_slab == NULL)
2599                 goto failure;
2600
2601         return base;
2602
2603 failure:
2604         if (!IS_ERR(clk)) {
2605                 clk_disable(clk);
2606                 clk_put(clk);
2607         }
2608         if (virtbase)
2609                 iounmap(virtbase);
2610         if (res)
2611                 release_mem_region(res->start,
2612                                    resource_size(res));
2613         if (virtbase)
2614                 iounmap(virtbase);
2615
2616         if (base) {
2617                 kfree(base->lcla_pool.alloc_map);
2618                 kfree(base->lookup_log_chans);
2619                 kfree(base->lookup_phy_chans);
2620                 kfree(base->phy_res);
2621                 kfree(base);
2622         }
2623
2624         return NULL;
2625 }
2626
2627 static void __init d40_hw_init(struct d40_base *base)
2628 {
2629
2630         static const struct d40_reg_val dma_init_reg[] = {
2631                 /* Clock every part of the DMA block from start */
2632                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2633
2634                 /* Interrupts on all logical channels */
2635                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2636                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2637                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2638                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2639                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2640                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2641                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2642                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2643                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2644                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2645                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2646                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2647         };
2648         int i;
2649         u32 prmseo[2] = {0, 0};
2650         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2651         u32 pcmis = 0;
2652         u32 pcicr = 0;
2653
2654         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2655                 writel(dma_init_reg[i].val,
2656                        base->virtbase + dma_init_reg[i].reg);
2657
2658         /* Configure all our dma channels to default settings */
2659         for (i = 0; i < base->num_phy_chans; i++) {
2660
2661                 activeo[i % 2] = activeo[i % 2] << 2;
2662
2663                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2664                     == D40_ALLOC_PHY) {
2665                         activeo[i % 2] |= 3;
2666                         continue;
2667                 }
2668
2669                 /* Enable interrupt # */
2670                 pcmis = (pcmis << 1) | 1;
2671
2672                 /* Clear interrupt # */
2673                 pcicr = (pcicr << 1) | 1;
2674
2675                 /* Set channel to physical mode */
2676                 prmseo[i % 2] = prmseo[i % 2] << 2;
2677                 prmseo[i % 2] |= 1;
2678
2679         }
2680
2681         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2682         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2683         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2684         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2685
2686         /* Write which interrupt to enable */
2687         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2688
2689         /* Write which interrupt to clear */
2690         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2691
2692 }
2693
2694 static int __init d40_lcla_allocate(struct d40_base *base)
2695 {
2696         struct d40_lcla_pool *pool = &base->lcla_pool;
2697         unsigned long *page_list;
2698         int i, j;
2699         int ret = 0;
2700
2701         /*
2702          * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2703          * To full fill this hardware requirement without wasting 256 kb
2704          * we allocate pages until we get an aligned one.
2705          */
2706         page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2707                             GFP_KERNEL);
2708
2709         if (!page_list) {
2710                 ret = -ENOMEM;
2711                 goto failure;
2712         }
2713
2714         /* Calculating how many pages that are required */
2715         base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2716
2717         for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2718                 page_list[i] = __get_free_pages(GFP_KERNEL,
2719                                                 base->lcla_pool.pages);
2720                 if (!page_list[i]) {
2721
2722                         d40_err(base->dev, "Failed to allocate %d pages.\n",
2723                                 base->lcla_pool.pages);
2724
2725                         for (j = 0; j < i; j++)
2726                                 free_pages(page_list[j], base->lcla_pool.pages);
2727                         goto failure;
2728                 }
2729
2730                 if ((virt_to_phys((void *)page_list[i]) &
2731                      (LCLA_ALIGNMENT - 1)) == 0)
2732                         break;
2733         }
2734
2735         for (j = 0; j < i; j++)
2736                 free_pages(page_list[j], base->lcla_pool.pages);
2737
2738         if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2739                 base->lcla_pool.base = (void *)page_list[i];
2740         } else {
2741                 /*
2742                  * After many attempts and no succees with finding the correct
2743                  * alignment, try with allocating a big buffer.
2744                  */
2745                 dev_warn(base->dev,
2746                          "[%s] Failed to get %d pages @ 18 bit align.\n",
2747                          __func__, base->lcla_pool.pages);
2748                 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2749                                                          base->num_phy_chans +
2750                                                          LCLA_ALIGNMENT,
2751                                                          GFP_KERNEL);
2752                 if (!base->lcla_pool.base_unaligned) {
2753                         ret = -ENOMEM;
2754                         goto failure;
2755                 }
2756
2757                 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2758                                                  LCLA_ALIGNMENT);
2759         }
2760
2761         pool->dma_addr = dma_map_single(base->dev, pool->base,
2762                                         SZ_1K * base->num_phy_chans,
2763                                         DMA_TO_DEVICE);
2764         if (dma_mapping_error(base->dev, pool->dma_addr)) {
2765                 pool->dma_addr = 0;
2766                 ret = -ENOMEM;
2767                 goto failure;
2768         }
2769
2770         writel(virt_to_phys(base->lcla_pool.base),
2771                base->virtbase + D40_DREG_LCLA);
2772 failure:
2773         kfree(page_list);
2774         return ret;
2775 }
2776
2777 static int __init d40_probe(struct platform_device *pdev)
2778 {
2779         int err;
2780         int ret = -ENOENT;
2781         struct d40_base *base;
2782         struct resource *res = NULL;
2783         int num_reserved_chans;
2784         u32 val;
2785
2786         base = d40_hw_detect_init(pdev);
2787
2788         if (!base)
2789                 goto failure;
2790
2791         num_reserved_chans = d40_phy_res_init(base);
2792
2793         platform_set_drvdata(pdev, base);
2794
2795         spin_lock_init(&base->interrupt_lock);
2796         spin_lock_init(&base->execmd_lock);
2797
2798         /* Get IO for logical channel parameter address */
2799         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2800         if (!res) {
2801                 ret = -ENOENT;
2802                 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
2803                 goto failure;
2804         }
2805         base->lcpa_size = resource_size(res);
2806         base->phy_lcpa = res->start;
2807
2808         if (request_mem_region(res->start, resource_size(res),
2809                                D40_NAME " I/O lcpa") == NULL) {
2810                 ret = -EBUSY;
2811                 d40_err(&pdev->dev,
2812                         "Failed to request LCPA region 0x%x-0x%x\n",
2813                         res->start, res->end);
2814                 goto failure;
2815         }
2816
2817         /* We make use of ESRAM memory for this. */
2818         val = readl(base->virtbase + D40_DREG_LCPA);
2819         if (res->start != val && val != 0) {
2820                 dev_warn(&pdev->dev,
2821                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2822                          __func__, val, res->start);
2823         } else
2824                 writel(res->start, base->virtbase + D40_DREG_LCPA);
2825
2826         base->lcpa_base = ioremap(res->start, resource_size(res));
2827         if (!base->lcpa_base) {
2828                 ret = -ENOMEM;
2829                 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
2830                 goto failure;
2831         }
2832
2833         ret = d40_lcla_allocate(base);
2834         if (ret) {
2835                 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
2836                 goto failure;
2837         }
2838
2839         spin_lock_init(&base->lcla_pool.lock);
2840
2841         base->irq = platform_get_irq(pdev, 0);
2842
2843         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2844         if (ret) {
2845                 d40_err(&pdev->dev, "No IRQ defined\n");
2846                 goto failure;
2847         }
2848
2849         err = d40_dmaengine_init(base, num_reserved_chans);
2850         if (err)
2851                 goto failure;
2852
2853         d40_hw_init(base);
2854
2855         dev_info(base->dev, "initialized\n");
2856         return 0;
2857
2858 failure:
2859         if (base) {
2860                 if (base->desc_slab)
2861                         kmem_cache_destroy(base->desc_slab);
2862                 if (base->virtbase)
2863                         iounmap(base->virtbase);
2864
2865                 if (base->lcla_pool.dma_addr)
2866                         dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2867                                          SZ_1K * base->num_phy_chans,
2868                                          DMA_TO_DEVICE);
2869
2870                 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2871                         free_pages((unsigned long)base->lcla_pool.base,
2872                                    base->lcla_pool.pages);
2873
2874                 kfree(base->lcla_pool.base_unaligned);
2875
2876                 if (base->phy_lcpa)
2877                         release_mem_region(base->phy_lcpa,
2878                                            base->lcpa_size);
2879                 if (base->phy_start)
2880                         release_mem_region(base->phy_start,
2881                                            base->phy_size);
2882                 if (base->clk) {
2883                         clk_disable(base->clk);
2884                         clk_put(base->clk);
2885                 }
2886
2887                 kfree(base->lcla_pool.alloc_map);
2888                 kfree(base->lookup_log_chans);
2889                 kfree(base->lookup_phy_chans);
2890                 kfree(base->phy_res);
2891                 kfree(base);
2892         }
2893
2894         d40_err(&pdev->dev, "probe failed\n");
2895         return ret;
2896 }
2897
2898 static struct platform_driver d40_driver = {
2899         .driver = {
2900                 .owner = THIS_MODULE,
2901                 .name  = D40_NAME,
2902         },
2903 };
2904
2905 static int __init stedma40_init(void)
2906 {
2907         return platform_driver_probe(&d40_driver, d40_probe);
2908 }
2909 arch_initcall(stedma40_init);