drivers/dma/ste_dma40.c

   1 /*
   2  * Copyright (C) Ericsson AB 2007-2008
   3  * Copyright (C) ST-Ericsson SA 2008-2010
   4  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
   5  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
   6  * License terms: GNU General Public License (GPL) version 2
   7  */
   8
   9 #include <linux/kernel.h>
  10 #include <linux/slab.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/clk.h>
  14 #include <linux/delay.h>
  15 #include <linux/err.h>
  16
  17 #include <plat/ste_dma40.h>
  18
  19 #include "ste_dma40_ll.h"
  20
  21 #define D40_NAME "dma40"
  22
  23 #define D40_PHY_CHAN -1
  24
  25 /* For masking out/in 2 bit channel positions */
  26 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
  27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  28
  29 /* Maximum iterations taken before giving up suspending a channel */
  30 #define D40_SUSPEND_MAX_IT 500
  31
  32 /* Hardware requirement on LCLA alignment */
  33 #define LCLA_ALIGNMENT 0x40000
  34
  35 /* Max number of links per event group */
  36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
  37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  38
  39 /* Attempts before giving up to trying to get pages that are aligned */
  40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
  41
  42 /* Bit markings for allocation map */
  43 #define D40_ALLOC_FREE          (1 << 31)
  44 #define D40_ALLOC_PHY           (1 << 30)
  45 #define D40_ALLOC_LOG_FREE      0
  46
  47 /* Hardware designer of the block */
  48 #define D40_HW_DESIGNER 0x8
  49
  50 /**
  51  * enum 40_command - The different commands and/or statuses.
  52  *
  53  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  54  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  55  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  56  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  57  */
  58 enum d40_command {
  59         D40_DMA_STOP            = 0,
  60         D40_DMA_RUN             = 1,
  61         D40_DMA_SUSPEND_REQ     = 2,
  62         D40_DMA_SUSPENDED       = 3
  63 };
  64
  65 /**
  66  * struct d40_lli_pool - Structure for keeping LLIs in memory
  67  *
  68  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  69  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  70  * pre_alloc_lli is used.
  71  * @dma_addr: DMA address, if mapped
  72  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  73  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  74  * one buffer to one buffer.
  75  */
  76 struct d40_lli_pool {
  77         void    *base;
  78         int      size;
  79         dma_addr_t      dma_addr;
  80         /* Space for dst and src, plus an extra for padding */
  81         u8       pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  82 };
  83
  84 /**
  85  * struct d40_desc - A descriptor is one DMA job.
  86  *
  87  * @lli_phy: LLI settings for physical channel. Both src and dst=
  88  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  89  * lli_len equals one.
  90  * @lli_log: Same as above but for logical channels.
  91  * @lli_pool: The pool with two entries pre-allocated.
  92  * @lli_len: Number of llis of current descriptor.
  93  * @lli_current: Number of transfered llis.
  94  * @lcla_alloc: Number of LCLA entries allocated.
  95  * @txd: DMA engine struct. Used for among other things for communication
  96  * during a transfer.
  97  * @node: List entry.
  98  * @is_in_client_list: true if the client owns this descriptor.
  99  * the previous one.
 100  *
 101  * This descriptor is used for both logical and physical transfers.
 102  */
 103 struct d40_desc {
 104         /* LLI physical */
 105         struct d40_phy_lli_bidir         lli_phy;
 106         /* LLI logical */
 107         struct d40_log_lli_bidir         lli_log;
 108
 109         struct d40_lli_pool              lli_pool;
 110         int                              lli_len;
 111         int                              lli_current;
 112         int                              lcla_alloc;
 113
 114         struct dma_async_tx_descriptor   txd;
 115         struct list_head                 node;
 116
 117         bool                             is_in_client_list;
 118 };
 119
 120 /**
 121  * struct d40_lcla_pool - LCLA pool settings and data.
 122  *
 123  * @base: The virtual address of LCLA. 18 bit aligned.
 124  * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
 125  * This pointer is only there for clean-up on error.
 126  * @pages: The number of pages needed for all physical channels.
 127  * Only used later for clean-up on error
 128  * @lock: Lock to protect the content in this struct.
 129  * @alloc_map: big map over which LCLA entry is own by which job.
 130  */
 131 struct d40_lcla_pool {
 132         void            *base;
 133         dma_addr_t      dma_addr;
 134         void            *base_unaligned;
 135         int              pages;
 136         spinlock_t       lock;
 137         struct d40_desc **alloc_map;
 138 };
 139
 140 /**
 141  * struct d40_phy_res - struct for handling eventlines mapped to physical
 142  * channels.
 143  *
 144  * @lock: A lock protection this entity.
 145  * @num: The physical channel number of this entity.
 146  * @allocated_src: Bit mapped to show which src event line's are mapped to
 147  * this physical channel. Can also be free or physically allocated.
 148  * @allocated_dst: Same as for src but is dst.
 149  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
 150  * event line number.
 151  */
 152 struct d40_phy_res {
 153         spinlock_t lock;
 154         int        num;
 155         u32        allocated_src;
 156         u32        allocated_dst;
 157 };
 158
 159 struct d40_base;
 160
 161 /**
 162  * struct d40_chan - Struct that describes a channel.
 163  *
 164  * @lock: A spinlock to protect this struct.
 165  * @log_num: The logical number, if any of this channel.
 166  * @completed: Starts with 1, after first interrupt it is set to dma engine's
 167  * current cookie.
 168  * @pending_tx: The number of pending transfers. Used between interrupt handler
 169  * and tasklet.
 170  * @busy: Set to true when transfer is ongoing on this channel.
 171  * @phy_chan: Pointer to physical channel which this instance runs on. If this
 172  * point is NULL, then the channel is not allocated.
 173  * @chan: DMA engine handle.
 174  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
 175  * transfer and call client callback.
 176  * @client: Cliented owned descriptor list.
 177  * @active: Active descriptor.
 178  * @queue: Queued jobs.
 179  * @dma_cfg: The client configuration of this dma channel.
 180  * @configured: whether the dma_cfg configuration is valid
 181  * @base: Pointer to the device instance struct.
 182  * @src_def_cfg: Default cfg register setting for src.
 183  * @dst_def_cfg: Default cfg register setting for dst.
 184  * @log_def: Default logical channel settings.
 185  * @lcla: Space for one dst src pair for logical channel transfers.
 186  * @lcpa: Pointer to dst and src lcpa settings.
 187  *
 188  * This struct can either "be" a logical or a physical channel.
 189  */
 190 struct d40_chan {
 191         spinlock_t                       lock;
 192         int                              log_num;
 193         /* ID of the most recent completed transfer */
 194         int                              completed;
 195         int                              pending_tx;
 196         bool                             busy;
 197         struct d40_phy_res              *phy_chan;
 198         struct dma_chan                  chan;
 199         struct tasklet_struct            tasklet;
 200         struct list_head                 client;
 201         struct list_head                 active;
 202         struct list_head                 queue;
 203         struct stedma40_chan_cfg         dma_cfg;
 204         bool                             configured;
 205         struct d40_base                 *base;
 206         /* Default register configurations */
 207         u32                              src_def_cfg;
 208         u32                              dst_def_cfg;
 209         struct d40_def_lcsp              log_def;
 210         struct d40_log_lli_full         *lcpa;
 211         /* Runtime reconfiguration */
 212         dma_addr_t                      runtime_addr;
 213         enum dma_data_direction         runtime_direction;
 214 };
 215
 216 /**
 217  * struct d40_base - The big global struct, one for each probe'd instance.
 218  *
 219  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
 220  * @execmd_lock: Lock for execute command usage since several channels share
 221  * the same physical register.
 222  * @dev: The device structure.
 223  * @virtbase: The virtual base address of the DMA's register.
 224  * @rev: silicon revision detected.
 225  * @clk: Pointer to the DMA clock structure.
 226  * @phy_start: Physical memory start of the DMA registers.
 227  * @phy_size: Size of the DMA register map.
 228  * @irq: The IRQ number.
 229  * @num_phy_chans: The number of physical channels. Read from HW. This
 230  * is the number of available channels for this driver, not counting "Secure
 231  * mode" allocated physical channels.
 232  * @num_log_chans: The number of logical channels. Calculated from
 233  * num_phy_chans.
 234  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
 235  * @dma_slave: dma_device channels that can do only do slave transfers.
 236  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
 237  * @log_chans: Room for all possible logical channels in system.
 238  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
 239  * to log_chans entries.
 240  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
 241  * to phy_chans entries.
 242  * @plat_data: Pointer to provided platform_data which is the driver
 243  * configuration.
 244  * @phy_res: Vector containing all physical channels.
 245  * @lcla_pool: lcla pool settings and data.
 246  * @lcpa_base: The virtual mapped address of LCPA.
 247  * @phy_lcpa: The physical address of the LCPA.
 248  * @lcpa_size: The size of the LCPA area.
 249  * @desc_slab: cache for descriptors.
 250  */
 251 struct d40_base {
 252         spinlock_t                       interrupt_lock;
 253         spinlock_t                       execmd_lock;
 254         struct device                    *dev;
 255         void __iomem                     *virtbase;
 256         u8                                rev:4;
 257         struct clk                       *clk;
 258         phys_addr_t                       phy_start;
 259         resource_size_t                   phy_size;
 260         int                               irq;
 261         int                               num_phy_chans;
 262         int                               num_log_chans;
 263         struct dma_device                 dma_both;
 264         struct dma_device                 dma_slave;
 265         struct dma_device                 dma_memcpy;
 266         struct d40_chan                  *phy_chans;
 267         struct d40_chan                  *log_chans;
 268         struct d40_chan                 **lookup_log_chans;
 269         struct d40_chan                 **lookup_phy_chans;
 270         struct stedma40_platform_data    *plat_data;
 271         /* Physical half channels */
 272         struct d40_phy_res               *phy_res;
 273         struct d40_lcla_pool              lcla_pool;
 274         void                             *lcpa_base;
 275         dma_addr_t                        phy_lcpa;
 276         resource_size_t                   lcpa_size;
 277         struct kmem_cache                *desc_slab;
 278 };
 279
 280 /**
 281  * struct d40_interrupt_lookup - lookup table for interrupt handler
 282  *
 283  * @src: Interrupt mask register.
 284  * @clr: Interrupt clear register.
 285  * @is_error: true if this is an error interrupt.
 286  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
 287  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
 288  */
 289 struct d40_interrupt_lookup {
 290         u32 src;
 291         u32 clr;
 292         bool is_error;
 293         int offset;
 294 };
 295
 296 /**
 297  * struct d40_reg_val - simple lookup struct
 298  *
 299  * @reg: The register.
 300  * @val: The value that belongs to the register in reg.
 301  */
 302 struct d40_reg_val {
 303         unsigned int reg;
 304         unsigned int val;
 305 };
 306
 307 static struct device *chan2dev(struct d40_chan *d40c)
 308 {
 309         return &d40c->chan.dev->device;
 310 }
 311
 312 static bool chan_is_physical(struct d40_chan *chan)
 313 {
 314         return chan->log_num == D40_PHY_CHAN;
 315 }
 316
 317 static bool chan_is_logical(struct d40_chan *chan)
 318 {
 319         return !chan_is_physical(chan);
 320 }
 321
 322 static void __iomem *chan_base(struct d40_chan *chan)
 323 {
 324         return chan->base->virtbase + D40_DREG_PCBASE +
 325                chan->phy_chan->num * D40_DREG_PCDELTA;
 326 }
 327
 328 #define d40_err(dev, format, arg...)            \
 329         dev_err(dev, "[%s] " format, __func__, ## arg)
 330
 331 #define chan_err(d40c, format, arg...)          \
 332         d40_err(chan2dev(d40c), format, ## arg)
 333
 334 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
 335                               int lli_len)
 336 {
 337         bool is_log = chan_is_logical(d40c);
 338         u32 align;
 339         void *base;
 340
 341         if (is_log)
 342                 align = sizeof(struct d40_log_lli);
 343         else
 344                 align = sizeof(struct d40_phy_lli);
 345
 346         if (lli_len == 1) {
 347                 base = d40d->lli_pool.pre_alloc_lli;
 348                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 349                 d40d->lli_pool.base = NULL;
 350         } else {
 351                 d40d->lli_pool.size = lli_len * 2 * align;
 352
 353                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 354                 d40d->lli_pool.base = base;
 355
 356                 if (d40d->lli_pool.base == NULL)
 357                         return -ENOMEM;
 358         }
 359
 360         if (is_log) {
 361                 d40d->lli_log.src = PTR_ALIGN(base, align);
 362                 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
 363
 364                 d40d->lli_pool.dma_addr = 0;
 365         } else {
 366                 d40d->lli_phy.src = PTR_ALIGN(base, align);
 367                 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
 368
 369                 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
 370                                                          d40d->lli_phy.src,
 371                                                          d40d->lli_pool.size,
 372                                                          DMA_TO_DEVICE);
 373
 374                 if (dma_mapping_error(d40c->base->dev,
 375                                       d40d->lli_pool.dma_addr)) {
 376                         kfree(d40d->lli_pool.base);
 377                         d40d->lli_pool.base = NULL;
 378                         d40d->lli_pool.dma_addr = 0;
 379                         return -ENOMEM;
 380                 }
 381         }
 382
 383         return 0;
 384 }
 385
 386 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
 387 {
 388         if (d40d->lli_pool.dma_addr)
 389                 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
 390                                  d40d->lli_pool.size, DMA_TO_DEVICE);
 391
 392         kfree(d40d->lli_pool.base);
 393         d40d->lli_pool.base = NULL;
 394         d40d->lli_pool.size = 0;
 395         d40d->lli_log.src = NULL;
 396         d40d->lli_log.dst = NULL;
 397         d40d->lli_phy.src = NULL;
 398         d40d->lli_phy.dst = NULL;
 399 }
 400
 401 static int d40_lcla_alloc_one(struct d40_chan *d40c,
 402                               struct d40_desc *d40d)
 403 {
 404         unsigned long flags;
 405         int i;
 406         int ret = -EINVAL;
 407         int p;
 408
 409         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 410
 411         p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
 412
 413         /*
 414          * Allocate both src and dst at the same time, therefore the half
 415          * start on 1 since 0 can't be used since zero is used as end marker.
 416          */
 417         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 418                 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
 419                         d40c->base->lcla_pool.alloc_map[p + i] = d40d;
 420                         d40d->lcla_alloc++;
 421                         ret = i;
 422                         break;
 423                 }
 424         }
 425
 426         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 427
 428         return ret;
 429 }
 430
 431 static int d40_lcla_free_all(struct d40_chan *d40c,
 432                              struct d40_desc *d40d)
 433 {
 434         unsigned long flags;
 435         int i;
 436         int ret = -EINVAL;
 437
 438         if (chan_is_physical(d40c))
 439                 return 0;
 440
 441         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 442
 443         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 444                 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 445                                                     D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
 446                         d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 447                                                         D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
 448                         d40d->lcla_alloc--;
 449                         if (d40d->lcla_alloc == 0) {
 450                                 ret = 0;
 451                                 break;
 452                         }
 453                 }
 454         }
 455
 456         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 457
 458         return ret;
 459
 460 }
 461
 462 static void d40_desc_remove(struct d40_desc *d40d)
 463 {
 464         list_del(&d40d->node);
 465 }
 466
 467 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 468 {
 469         struct d40_desc *desc = NULL;
 470
 471         if (!list_empty(&d40c->client)) {
 472                 struct d40_desc *d;
 473                 struct d40_desc *_d;
 474
 475                 list_for_each_entry_safe(d, _d, &d40c->client, node)
 476                         if (async_tx_test_ack(&d->txd)) {
 477                                 d40_pool_lli_free(d40c, d);
 478                                 d40_desc_remove(d);
 479                                 desc = d;
 480                                 memset(desc, 0, sizeof(*desc));
 481                                 break;
 482                         }
 483         }
 484
 485         if (!desc)
 486                 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
 487
 488         if (desc)
 489                 INIT_LIST_HEAD(&desc->node);
 490
 491         return desc;
 492 }
 493
 494 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 495 {
 496
 497         d40_pool_lli_free(d40c, d40d);
 498         d40_lcla_free_all(d40c, d40d);
 499         kmem_cache_free(d40c->base->desc_slab, d40d);
 500 }
 501
 502 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 503 {
 504         list_add_tail(&desc->node, &d40c->active);
 505 }
 506
 507 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
 508 {
 509         struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
 510         struct d40_phy_lli *lli_src = desc->lli_phy.src;
 511         void __iomem *base = chan_base(chan);
 512
 513         writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
 514         writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
 515         writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
 516         writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
 517
 518         writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
 519         writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
 520         writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
 521         writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
 522 }
 523
 524 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 525 {
 526         int curr_lcla = -EINVAL, next_lcla;
 527
 528         if (chan_is_physical(d40c)) {
 529                 d40_phy_lli_load(d40c, d40d);
 530                 d40d->lli_current = d40d->lli_len;
 531         } else {
 532
 533                 if ((d40d->lli_len - d40d->lli_current) > 1)
 534                         curr_lcla = d40_lcla_alloc_one(d40c, d40d);
 535
 536                 d40_log_lli_lcpa_write(d40c->lcpa,
 537                                        &d40d->lli_log.dst[d40d->lli_current],
 538                                        &d40d->lli_log.src[d40d->lli_current],
 539                                        curr_lcla);
 540
 541                 d40d->lli_current++;
 542                 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
 543                         unsigned int lcla_offset = d40c->phy_chan->num * 1024 +
 544                                                    8 * curr_lcla * 2;
 545                         struct d40_lcla_pool *pool = &d40c->base->lcla_pool;
 546                         struct d40_log_lli *lcla = pool->base + lcla_offset;
 547
 548                         if (d40d->lli_current + 1 < d40d->lli_len)
 549                                 next_lcla = d40_lcla_alloc_one(d40c, d40d);
 550                         else
 551                                 next_lcla = -EINVAL;
 552
 553                         d40_log_lli_lcla_write(lcla,
 554                                                &d40d->lli_log.dst[d40d->lli_current],
 555                                                &d40d->lli_log.src[d40d->lli_current],
 556                                                next_lcla);
 557
 558                         dma_sync_single_range_for_device(d40c->base->dev,
 559                                                 pool->dma_addr, lcla_offset,
 560                                                 2 * sizeof(struct d40_log_lli),
 561                                                 DMA_TO_DEVICE);
 562
 563                         curr_lcla = next_lcla;
 564
 565                         if (curr_lcla == -EINVAL) {
 566                                 d40d->lli_current++;
 567                                 break;
 568                         }
 569
 570                 }
 571         }
 572 }
 573
 574 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 575 {
 576         struct d40_desc *d;
 577
 578         if (list_empty(&d40c->active))
 579                 return NULL;
 580
 581         d = list_first_entry(&d40c->active,
 582                              struct d40_desc,
 583                              node);
 584         return d;
 585 }
 586
 587 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 588 {
 589         list_add_tail(&desc->node, &d40c->queue);
 590 }
 591
 592 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 593 {
 594         struct d40_desc *d;
 595
 596         if (list_empty(&d40c->queue))
 597                 return NULL;
 598
 599         d = list_first_entry(&d40c->queue,
 600                              struct d40_desc,
 601                              node);
 602         return d;
 603 }
 604
 605 static int d40_psize_2_burst_size(bool is_log, int psize)
 606 {
 607         if (is_log) {
 608                 if (psize == STEDMA40_PSIZE_LOG_1)
 609                         return 1;
 610         } else {
 611                 if (psize == STEDMA40_PSIZE_PHY_1)
 612                         return 1;
 613         }
 614
 615         return 2 << psize;
 616 }
 617
 618 /*
 619  * The dma only supports transmitting packages up to
 620  * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
 621  * dma elements required to send the entire sg list
 622  */
 623 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
 624 {
 625         int dmalen;
 626         u32 max_w = max(data_width1, data_width2);
 627         u32 min_w = min(data_width1, data_width2);
 628         u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
 629
 630         if (seg_max > STEDMA40_MAX_SEG_SIZE)
 631                 seg_max -= (1 << max_w);
 632
 633         if (!IS_ALIGNED(size, 1 << max_w))
 634                 return -EINVAL;
 635
 636         if (size <= seg_max)
 637                 dmalen = 1;
 638         else {
 639                 dmalen = size / seg_max;
 640                 if (dmalen * seg_max < size)
 641                         dmalen++;
 642         }
 643         return dmalen;
 644 }
 645
 646 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
 647                            u32 data_width1, u32 data_width2)
 648 {
 649         struct scatterlist *sg;
 650         int i;
 651         int len = 0;
 652         int ret;
 653
 654         for_each_sg(sgl, sg, sg_len, i) {
 655                 ret = d40_size_2_dmalen(sg_dma_len(sg),
 656                                         data_width1, data_width2);
 657                 if (ret < 0)
 658                         return ret;
 659                 len += ret;
 660         }
 661         return len;
 662 }
 663
 664 /* Support functions for logical channels */
 665
 666 static int d40_channel_execute_command(struct d40_chan *d40c,
 667                                        enum d40_command command)
 668 {
 669         u32 status;
 670         int i;
 671         void __iomem *active_reg;
 672         int ret = 0;
 673         unsigned long flags;
 674         u32 wmask;
 675
 676         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 677
 678         if (d40c->phy_chan->num % 2 == 0)
 679                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 680         else
 681                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 682
 683         if (command == D40_DMA_SUSPEND_REQ) {
 684                 status = (readl(active_reg) &
 685                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 686                         D40_CHAN_POS(d40c->phy_chan->num);
 687
 688                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 689                         goto done;
 690         }
 691
 692         wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
 693         writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
 694                active_reg);
 695
 696         if (command == D40_DMA_SUSPEND_REQ) {
 697
 698                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 699                         status = (readl(active_reg) &
 700                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 701                                 D40_CHAN_POS(d40c->phy_chan->num);
 702
 703                         cpu_relax();
 704                         /*
 705                          * Reduce the number of bus accesses while
 706                          * waiting for the DMA to suspend.
 707                          */
 708                         udelay(3);
 709
 710                         if (status == D40_DMA_STOP ||
 711                             status == D40_DMA_SUSPENDED)
 712                                 break;
 713                 }
 714
 715                 if (i == D40_SUSPEND_MAX_IT) {
 716                         chan_err(d40c,
 717                                 "unable to suspend the chl %d (log: %d) status %x\n",
 718                                 d40c->phy_chan->num, d40c->log_num,
 719                                 status);
 720                         dump_stack();
 721                         ret = -EBUSY;
 722                 }
 723
 724         }
 725 done:
 726         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 727         return ret;
 728 }
 729
 730 static void d40_term_all(struct d40_chan *d40c)
 731 {
 732         struct d40_desc *d40d;
 733
 734         /* Release active descriptors */
 735         while ((d40d = d40_first_active_get(d40c))) {
 736                 d40_desc_remove(d40d);
 737                 d40_desc_free(d40c, d40d);
 738         }
 739
 740         /* Release queued descriptors waiting for transfer */
 741         while ((d40d = d40_first_queued(d40c))) {
 742                 d40_desc_remove(d40d);
 743                 d40_desc_free(d40c, d40d);
 744         }
 745
 746
 747         d40c->pending_tx = 0;
 748         d40c->busy = false;
 749 }
 750
 751 static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
 752                                    u32 event, int reg)
 753 {
 754         void __iomem *addr = chan_base(d40c) + reg;
 755         int tries;
 756
 757         if (!enable) {
 758                 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 759                        | ~D40_EVENTLINE_MASK(event), addr);
 760                 return;
 761         }
 762
 763         /*
 764          * The hardware sometimes doesn't register the enable when src and dst
 765          * event lines are active on the same logical channel.  Retry to ensure
 766          * it does.  Usually only one retry is sufficient.
 767          */
 768         tries = 100;
 769         while (--tries) {
 770                 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 771                        | ~D40_EVENTLINE_MASK(event), addr);
 772
 773                 if (readl(addr) & D40_EVENTLINE_MASK(event))
 774                         break;
 775         }
 776
 777         if (tries != 99)
 778                 dev_dbg(chan2dev(d40c),
 779                         "[%s] workaround enable S%cLNK (%d tries)\n",
 780                         __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
 781                         100 - tries);
 782
 783         WARN_ON(!tries);
 784 }
 785
 786 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 787 {
 788         unsigned long flags;
 789
 790         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 791
 792         /* Enable event line connected to device (or memcpy) */
 793         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 794             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 795                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 796
 797                 __d40_config_set_event(d40c, do_enable, event,
 798                                        D40_CHAN_REG_SSLNK);
 799         }
 800
 801         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 802                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 803
 804                 __d40_config_set_event(d40c, do_enable, event,
 805                                        D40_CHAN_REG_SDLNK);
 806         }
 807
 808         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 809 }
 810
 811 static u32 d40_chan_has_events(struct d40_chan *d40c)
 812 {
 813         void __iomem *chanbase = chan_base(d40c);
 814         u32 val;
 815
 816         val = readl(chanbase + D40_CHAN_REG_SSLNK);
 817         val |= readl(chanbase + D40_CHAN_REG_SDLNK);
 818
 819         return val;
 820 }
 821
 822 static u32 d40_get_prmo(struct d40_chan *d40c)
 823 {
 824         static const unsigned int phy_map[] = {
 825                 [STEDMA40_PCHAN_BASIC_MODE]
 826                         = D40_DREG_PRMO_PCHAN_BASIC,
 827                 [STEDMA40_PCHAN_MODULO_MODE]
 828                         = D40_DREG_PRMO_PCHAN_MODULO,
 829                 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
 830                         = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
 831         };
 832         static const unsigned int log_map[] = {
 833                 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
 834                         = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
 835                 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
 836                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
 837                 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
 838                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
 839         };
 840
 841         if (chan_is_physical(d40c))
 842                 return phy_map[d40c->dma_cfg.mode_opt];
 843         else
 844                 return log_map[d40c->dma_cfg.mode_opt];
 845 }
 846
 847 static void d40_config_write(struct d40_chan *d40c)
 848 {
 849         u32 addr_base;
 850         u32 var;
 851
 852         /* Odd addresses are even addresses + 4 */
 853         addr_base = (d40c->phy_chan->num % 2) * 4;
 854         /* Setup channel mode to logical or physical */
 855         var = ((u32)(chan_is_logical(d40c)) + 1) <<
 856                 D40_CHAN_POS(d40c->phy_chan->num);
 857         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 858
 859         /* Setup operational mode option register */
 860         var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
 861
 862         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 863
 864         if (chan_is_logical(d40c)) {
 865                 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
 866                            & D40_SREG_ELEM_LOG_LIDX_MASK;
 867                 void __iomem *chanbase = chan_base(d40c);
 868
 869                 /* Set default config for CFG reg */
 870                 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
 871                 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
 872
 873                 /* Set LIDX for lcla */
 874                 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
 875                 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
 876         }
 877 }
 878
 879 static u32 d40_residue(struct d40_chan *d40c)
 880 {
 881         u32 num_elt;
 882
 883         if (chan_is_logical(d40c))
 884                 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
 885                         >> D40_MEM_LCSP2_ECNT_POS;
 886         else {
 887                 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
 888                 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
 889                           >> D40_SREG_ELEM_PHY_ECNT_POS;
 890         }
 891
 892         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
 893 }
 894
 895 static bool d40_tx_is_linked(struct d40_chan *d40c)
 896 {
 897         bool is_link;
 898
 899         if (chan_is_logical(d40c))
 900                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
 901         else
 902                 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
 903                           & D40_SREG_LNK_PHYS_LNK_MASK;
 904
 905         return is_link;
 906 }
 907
 908 static int d40_pause(struct dma_chan *chan)
 909 {
 910         struct d40_chan *d40c =
 911                 container_of(chan, struct d40_chan, chan);
 912         int res = 0;
 913         unsigned long flags;
 914
 915         if (!d40c->busy)
 916                 return 0;
 917
 918         spin_lock_irqsave(&d40c->lock, flags);
 919
 920         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 921         if (res == 0) {
 922                 if (chan_is_logical(d40c)) {
 923                         d40_config_set_event(d40c, false);
 924                         /* Resume the other logical channels if any */
 925                         if (d40_chan_has_events(d40c))
 926                                 res = d40_channel_execute_command(d40c,
 927                                                                   D40_DMA_RUN);
 928                 }
 929         }
 930
 931         spin_unlock_irqrestore(&d40c->lock, flags);
 932         return res;
 933 }
 934
 935 static int d40_resume(struct dma_chan *chan)
 936 {
 937         struct d40_chan *d40c =
 938                 container_of(chan, struct d40_chan, chan);
 939         int res = 0;
 940         unsigned long flags;
 941
 942         if (!d40c->busy)
 943                 return 0;
 944
 945         spin_lock_irqsave(&d40c->lock, flags);
 946
 947         if (d40c->base->rev == 0)
 948                 if (chan_is_logical(d40c)) {
 949                         res = d40_channel_execute_command(d40c,
 950                                                           D40_DMA_SUSPEND_REQ);
 951                         goto no_suspend;
 952                 }
 953
 954         /* If bytes left to transfer or linked tx resume job */
 955         if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
 956
 957                 if (chan_is_logical(d40c))
 958                         d40_config_set_event(d40c, true);
 959
 960                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 961         }
 962
 963 no_suspend:
 964         spin_unlock_irqrestore(&d40c->lock, flags);
 965         return res;
 966 }
 967
 968 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
 969 {
 970         struct d40_chan *d40c = container_of(tx->chan,
 971                                              struct d40_chan,
 972                                              chan);
 973         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
 974         unsigned long flags;
 975
 976         spin_lock_irqsave(&d40c->lock, flags);
 977
 978         d40c->chan.cookie++;
 979
 980         if (d40c->chan.cookie < 0)
 981                 d40c->chan.cookie = 1;
 982
 983         d40d->txd.cookie = d40c->chan.cookie;
 984
 985         d40_desc_queue(d40c, d40d);
 986
 987         spin_unlock_irqrestore(&d40c->lock, flags);
 988
 989         return tx->cookie;
 990 }
 991
 992 static int d40_start(struct d40_chan *d40c)
 993 {
 994         if (d40c->base->rev == 0) {
 995                 int err;
 996
 997                 if (chan_is_logical(d40c)) {
 998                         err = d40_channel_execute_command(d40c,
 999                                                           D40_DMA_SUSPEND_REQ);
1000                         if (err)
1001                                 return err;
1002                 }
1003         }
1004
1005         if (chan_is_logical(d40c))
1006                 d40_config_set_event(d40c, true);
1007
1008         return d40_channel_execute_command(d40c, D40_DMA_RUN);
1009 }
1010
1011 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1012 {
1013         struct d40_desc *d40d;
1014         int err;
1015
1016         /* Start queued jobs, if any */
1017         d40d = d40_first_queued(d40c);
1018
1019         if (d40d != NULL) {
1020                 d40c->busy = true;
1021
1022                 /* Remove from queue */
1023                 d40_desc_remove(d40d);
1024
1025                 /* Add to active queue */
1026                 d40_desc_submit(d40c, d40d);
1027
1028                 /* Initiate DMA job */
1029                 d40_desc_load(d40c, d40d);
1030
1031                 /* Start dma job */
1032                 err = d40_start(d40c);
1033
1034                 if (err)
1035                         return NULL;
1036         }
1037
1038         return d40d;
1039 }
1040
1041 /* called from interrupt context */
1042 static void dma_tc_handle(struct d40_chan *d40c)
1043 {
1044         struct d40_desc *d40d;
1045
1046         /* Get first active entry from list */
1047         d40d = d40_first_active_get(d40c);
1048
1049         if (d40d == NULL)
1050                 return;
1051
1052         d40_lcla_free_all(d40c, d40d);
1053
1054         if (d40d->lli_current < d40d->lli_len) {
1055                 d40_desc_load(d40c, d40d);
1056                 /* Start dma job */
1057                 (void) d40_start(d40c);
1058                 return;
1059         }
1060
1061         if (d40_queue_start(d40c) == NULL)
1062                 d40c->busy = false;
1063
1064         d40c->pending_tx++;
1065         tasklet_schedule(&d40c->tasklet);
1066
1067 }
1068
1069 static void dma_tasklet(unsigned long data)
1070 {
1071         struct d40_chan *d40c = (struct d40_chan *) data;
1072         struct d40_desc *d40d;
1073         unsigned long flags;
1074         dma_async_tx_callback callback;
1075         void *callback_param;
1076
1077         spin_lock_irqsave(&d40c->lock, flags);
1078
1079         /* Get first active entry from list */
1080         d40d = d40_first_active_get(d40c);
1081
1082         if (d40d == NULL)
1083                 goto err;
1084
1085         d40c->completed = d40d->txd.cookie;
1086
1087         /*
1088          * If terminating a channel pending_tx is set to zero.
1089          * This prevents any finished active jobs to return to the client.
1090          */
1091         if (d40c->pending_tx == 0) {
1092                 spin_unlock_irqrestore(&d40c->lock, flags);
1093                 return;
1094         }
1095
1096         /* Callback to client */
1097         callback = d40d->txd.callback;
1098         callback_param = d40d->txd.callback_param;
1099
1100         if (async_tx_test_ack(&d40d->txd)) {
1101                 d40_pool_lli_free(d40c, d40d);
1102                 d40_desc_remove(d40d);
1103                 d40_desc_free(d40c, d40d);
1104         } else {
1105                 if (!d40d->is_in_client_list) {
1106                         d40_desc_remove(d40d);
1107                         d40_lcla_free_all(d40c, d40d);
1108                         list_add_tail(&d40d->node, &d40c->client);
1109                         d40d->is_in_client_list = true;
1110                 }
1111         }
1112
1113         d40c->pending_tx--;
1114
1115         if (d40c->pending_tx)
1116                 tasklet_schedule(&d40c->tasklet);
1117
1118         spin_unlock_irqrestore(&d40c->lock, flags);
1119
1120         if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1121                 callback(callback_param);
1122
1123         return;
1124
1125  err:
1126         /* Rescue manouver if receiving double interrupts */
1127         if (d40c->pending_tx > 0)
1128                 d40c->pending_tx--;
1129         spin_unlock_irqrestore(&d40c->lock, flags);
1130 }
1131
1132 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1133 {
1134         static const struct d40_interrupt_lookup il[] = {
1135                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
1136                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1137                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1138                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1139                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
1140                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
1141                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
1142                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
1143                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
1144                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
1145         };
1146
1147         int i;
1148         u32 regs[ARRAY_SIZE(il)];
1149         u32 idx;
1150         u32 row;
1151         long chan = -1;
1152         struct d40_chan *d40c;
1153         unsigned long flags;
1154         struct d40_base *base = data;
1155
1156         spin_lock_irqsave(&base->interrupt_lock, flags);
1157
1158         /* Read interrupt status of both logical and physical channels */
1159         for (i = 0; i < ARRAY_SIZE(il); i++)
1160                 regs[i] = readl(base->virtbase + il[i].src);
1161
1162         for (;;) {
1163
1164                 chan = find_next_bit((unsigned long *)regs,
1165                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1166
1167                 /* No more set bits found? */
1168                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1169                         break;
1170
1171                 row = chan / BITS_PER_LONG;
1172                 idx = chan & (BITS_PER_LONG - 1);
1173
1174                 /* ACK interrupt */
1175                 writel(1 << idx, base->virtbase + il[row].clr);
1176
1177                 if (il[row].offset == D40_PHY_CHAN)
1178                         d40c = base->lookup_phy_chans[idx];
1179                 else
1180                         d40c = base->lookup_log_chans[il[row].offset + idx];
1181                 spin_lock(&d40c->lock);
1182
1183                 if (!il[row].is_error)
1184                         dma_tc_handle(d40c);
1185                 else
1186                         d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1187                                 chan, il[row].offset, idx);
1188
1189                 spin_unlock(&d40c->lock);
1190         }
1191
1192         spin_unlock_irqrestore(&base->interrupt_lock, flags);
1193
1194         return IRQ_HANDLED;
1195 }
1196
1197 static int d40_validate_conf(struct d40_chan *d40c,
1198                              struct stedma40_chan_cfg *conf)
1199 {
1200         int res = 0;
1201         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1202         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1203         bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1204
1205         if (!conf->dir) {
1206                 chan_err(d40c, "Invalid direction.\n");
1207                 res = -EINVAL;
1208         }
1209
1210         if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1211             d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1212             d40c->runtime_addr == 0) {
1213
1214                 chan_err(d40c, "Invalid TX channel address (%d)\n",
1215                          conf->dst_dev_type);
1216                 res = -EINVAL;
1217         }
1218
1219         if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1220             d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1221             d40c->runtime_addr == 0) {
1222                 chan_err(d40c, "Invalid RX channel address (%d)\n",
1223                         conf->src_dev_type);
1224                 res = -EINVAL;
1225         }
1226
1227         if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1228             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1229                 chan_err(d40c, "Invalid dst\n");
1230                 res = -EINVAL;
1231         }
1232
1233         if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1234             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1235                 chan_err(d40c, "Invalid src\n");
1236                 res = -EINVAL;
1237         }
1238
1239         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1240             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1241                 chan_err(d40c, "No event line\n");
1242                 res = -EINVAL;
1243         }
1244
1245         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1246             (src_event_group != dst_event_group)) {
1247                 chan_err(d40c, "Invalid event group\n");
1248                 res = -EINVAL;
1249         }
1250
1251         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1252                 /*
1253                  * DMAC HW supports it. Will be added to this driver,
1254                  * in case any dma client requires it.
1255                  */
1256                 chan_err(d40c, "periph to periph not supported\n");
1257                 res = -EINVAL;
1258         }
1259
1260         if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1261             (1 << conf->src_info.data_width) !=
1262             d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1263             (1 << conf->dst_info.data_width)) {
1264                 /*
1265                  * The DMAC hardware only supports
1266                  * src (burst x width) == dst (burst x width)
1267                  */
1268
1269                 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1270                 res = -EINVAL;
1271         }
1272
1273         return res;
1274 }
1275
1276 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1277                                int log_event_line, bool is_log)
1278 {
1279         unsigned long flags;
1280         spin_lock_irqsave(&phy->lock, flags);
1281         if (!is_log) {
1282                 /* Physical interrupts are masked per physical full channel */
1283                 if (phy->allocated_src == D40_ALLOC_FREE &&
1284                     phy->allocated_dst == D40_ALLOC_FREE) {
1285                         phy->allocated_dst = D40_ALLOC_PHY;
1286                         phy->allocated_src = D40_ALLOC_PHY;
1287                         goto found;
1288                 } else
1289                         goto not_found;
1290         }
1291
1292         /* Logical channel */
1293         if (is_src) {
1294                 if (phy->allocated_src == D40_ALLOC_PHY)
1295                         goto not_found;
1296
1297                 if (phy->allocated_src == D40_ALLOC_FREE)
1298                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1299
1300                 if (!(phy->allocated_src & (1 << log_event_line))) {
1301                         phy->allocated_src |= 1 << log_event_line;
1302                         goto found;
1303                 } else
1304                         goto not_found;
1305         } else {
1306                 if (phy->allocated_dst == D40_ALLOC_PHY)
1307                         goto not_found;
1308
1309                 if (phy->allocated_dst == D40_ALLOC_FREE)
1310                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1311
1312                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1313                         phy->allocated_dst |= 1 << log_event_line;
1314                         goto found;
1315                 } else
1316                         goto not_found;
1317         }
1318
1319 not_found:
1320         spin_unlock_irqrestore(&phy->lock, flags);
1321         return false;
1322 found:
1323         spin_unlock_irqrestore(&phy->lock, flags);
1324         return true;
1325 }
1326
1327 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1328                                int log_event_line)
1329 {
1330         unsigned long flags;
1331         bool is_free = false;
1332
1333         spin_lock_irqsave(&phy->lock, flags);
1334         if (!log_event_line) {
1335                 phy->allocated_dst = D40_ALLOC_FREE;
1336                 phy->allocated_src = D40_ALLOC_FREE;
1337                 is_free = true;
1338                 goto out;
1339         }
1340
1341         /* Logical channel */
1342         if (is_src) {
1343                 phy->allocated_src &= ~(1 << log_event_line);
1344                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1345                         phy->allocated_src = D40_ALLOC_FREE;
1346         } else {
1347                 phy->allocated_dst &= ~(1 << log_event_line);
1348                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1349                         phy->allocated_dst = D40_ALLOC_FREE;
1350         }
1351
1352         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1353                    D40_ALLOC_FREE);
1354
1355 out:
1356         spin_unlock_irqrestore(&phy->lock, flags);
1357
1358         return is_free;
1359 }
1360
1361 static int d40_allocate_channel(struct d40_chan *d40c)
1362 {
1363         int dev_type;
1364         int event_group;
1365         int event_line;
1366         struct d40_phy_res *phys;
1367         int i;
1368         int j;
1369         int log_num;
1370         bool is_src;
1371         bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1372
1373         phys = d40c->base->phy_res;
1374
1375         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1376                 dev_type = d40c->dma_cfg.src_dev_type;
1377                 log_num = 2 * dev_type;
1378                 is_src = true;
1379         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1380                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1381                 /* dst event lines are used for logical memcpy */
1382                 dev_type = d40c->dma_cfg.dst_dev_type;
1383                 log_num = 2 * dev_type + 1;
1384                 is_src = false;
1385         } else
1386                 return -EINVAL;
1387
1388         event_group = D40_TYPE_TO_GROUP(dev_type);
1389         event_line = D40_TYPE_TO_EVENT(dev_type);
1390
1391         if (!is_log) {
1392                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1393                         /* Find physical half channel */
1394                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1395
1396                                 if (d40_alloc_mask_set(&phys[i], is_src,
1397                                                        0, is_log))
1398                                         goto found_phy;
1399                         }
1400                 } else
1401                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1402                                 int phy_num = j  + event_group * 2;
1403                                 for (i = phy_num; i < phy_num + 2; i++) {
1404                                         if (d40_alloc_mask_set(&phys[i],
1405                                                                is_src,
1406                                                                0,
1407                                                                is_log))
1408                                                 goto found_phy;
1409                                 }
1410                         }
1411                 return -EINVAL;
1412 found_phy:
1413                 d40c->phy_chan = &phys[i];
1414                 d40c->log_num = D40_PHY_CHAN;
1415                 goto out;
1416         }
1417         if (dev_type == -1)
1418                 return -EINVAL;
1419
1420         /* Find logical channel */
1421         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1422                 int phy_num = j + event_group * 2;
1423                 /*
1424                  * Spread logical channels across all available physical rather
1425                  * than pack every logical channel at the first available phy
1426                  * channels.
1427                  */
1428                 if (is_src) {
1429                         for (i = phy_num; i < phy_num + 2; i++) {
1430                                 if (d40_alloc_mask_set(&phys[i], is_src,
1431                                                        event_line, is_log))
1432                                         goto found_log;
1433                         }
1434                 } else {
1435                         for (i = phy_num + 1; i >= phy_num; i--) {
1436                                 if (d40_alloc_mask_set(&phys[i], is_src,
1437                                                        event_line, is_log))
1438                                         goto found_log;
1439                         }
1440                 }
1441         }
1442         return -EINVAL;
1443
1444 found_log:
1445         d40c->phy_chan = &phys[i];
1446         d40c->log_num = log_num;
1447 out:
1448
1449         if (is_log)
1450                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1451         else
1452                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1453
1454         return 0;
1455
1456 }
1457
1458 static int d40_config_memcpy(struct d40_chan *d40c)
1459 {
1460         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1461
1462         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1463                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1464                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1465                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1466                         memcpy[d40c->chan.chan_id];
1467
1468         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1469                    dma_has_cap(DMA_SLAVE, cap)) {
1470                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1471         } else {
1472                 chan_err(d40c, "No memcpy\n");
1473                 return -EINVAL;
1474         }
1475
1476         return 0;
1477 }
1478
1479
1480 static int d40_free_dma(struct d40_chan *d40c)
1481 {
1482
1483         int res = 0;
1484         u32 event;
1485         struct d40_phy_res *phy = d40c->phy_chan;
1486         bool is_src;
1487         struct d40_desc *d;
1488         struct d40_desc *_d;
1489
1490
1491         /* Terminate all queued and active transfers */
1492         d40_term_all(d40c);
1493
1494         /* Release client owned descriptors */
1495         if (!list_empty(&d40c->client))
1496                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1497                         d40_pool_lli_free(d40c, d);
1498                         d40_desc_remove(d);
1499                         d40_desc_free(d40c, d);
1500                 }
1501
1502         if (phy == NULL) {
1503                 chan_err(d40c, "phy == null\n");
1504                 return -EINVAL;
1505         }
1506
1507         if (phy->allocated_src == D40_ALLOC_FREE &&
1508             phy->allocated_dst == D40_ALLOC_FREE) {
1509                 chan_err(d40c, "channel already free\n");
1510                 return -EINVAL;
1511         }
1512
1513         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1514             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1515                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1516                 is_src = false;
1517         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1518                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1519                 is_src = true;
1520         } else {
1521                 chan_err(d40c, "Unknown direction\n");
1522                 return -EINVAL;
1523         }
1524
1525         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1526         if (res) {
1527                 chan_err(d40c, "suspend failed\n");
1528                 return res;
1529         }
1530
1531         if (chan_is_logical(d40c)) {
1532                 /* Release logical channel, deactivate the event line */
1533
1534                 d40_config_set_event(d40c, false);
1535                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1536
1537                 /*
1538                  * Check if there are more logical allocation
1539                  * on this phy channel.
1540                  */
1541                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1542                         /* Resume the other logical channels if any */
1543                         if (d40_chan_has_events(d40c)) {
1544                                 res = d40_channel_execute_command(d40c,
1545                                                                   D40_DMA_RUN);
1546                                 if (res) {
1547                                         chan_err(d40c,
1548                                                 "Executing RUN command\n");
1549                                         return res;
1550                                 }
1551                         }
1552                         return 0;
1553                 }
1554         } else {
1555                 (void) d40_alloc_mask_free(phy, is_src, 0);
1556         }
1557
1558         /* Release physical channel */
1559         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1560         if (res) {
1561                 chan_err(d40c, "Failed to stop channel\n");
1562                 return res;
1563         }
1564         d40c->phy_chan = NULL;
1565         d40c->configured = false;
1566         d40c->base->lookup_phy_chans[phy->num] = NULL;
1567
1568         return 0;
1569 }
1570
1571 static bool d40_is_paused(struct d40_chan *d40c)
1572 {
1573         void __iomem *chanbase = chan_base(d40c);
1574         bool is_paused = false;
1575         unsigned long flags;
1576         void __iomem *active_reg;
1577         u32 status;
1578         u32 event;
1579
1580         spin_lock_irqsave(&d40c->lock, flags);
1581
1582         if (chan_is_physical(d40c)) {
1583                 if (d40c->phy_chan->num % 2 == 0)
1584                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1585                 else
1586                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1587
1588                 status = (readl(active_reg) &
1589                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1590                         D40_CHAN_POS(d40c->phy_chan->num);
1591                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1592                         is_paused = true;
1593
1594                 goto _exit;
1595         }
1596
1597         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1598             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1599                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1600                 status = readl(chanbase + D40_CHAN_REG_SDLNK);
1601         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1602                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1603                 status = readl(chanbase + D40_CHAN_REG_SSLNK);
1604         } else {
1605                 chan_err(d40c, "Unknown direction\n");
1606                 goto _exit;
1607         }
1608
1609         status = (status & D40_EVENTLINE_MASK(event)) >>
1610                 D40_EVENTLINE_POS(event);
1611
1612         if (status != D40_DMA_RUN)
1613                 is_paused = true;
1614 _exit:
1615         spin_unlock_irqrestore(&d40c->lock, flags);
1616         return is_paused;
1617
1618 }
1619
1620
1621 static u32 stedma40_residue(struct dma_chan *chan)
1622 {
1623         struct d40_chan *d40c =
1624                 container_of(chan, struct d40_chan, chan);
1625         u32 bytes_left;
1626         unsigned long flags;
1627
1628         spin_lock_irqsave(&d40c->lock, flags);
1629         bytes_left = d40_residue(d40c);
1630         spin_unlock_irqrestore(&d40c->lock, flags);
1631
1632         return bytes_left;
1633 }
1634
1635 static int
1636 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
1637                 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1638                 unsigned int sg_len, enum dma_data_direction direction,
1639                 dma_addr_t dev_addr)
1640 {
1641         dma_addr_t src_dev_addr = direction == DMA_FROM_DEVICE ? dev_addr : 0;
1642         dma_addr_t dst_dev_addr = direction == DMA_TO_DEVICE ? dev_addr : 0;
1643         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1644         struct stedma40_half_channel_info *src_info = &cfg->src_info;
1645         struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1646         int ret;
1647
1648         ret = d40_log_sg_to_lli(sg_src, sg_len,
1649                                 src_dev_addr,
1650                                 desc->lli_log.src,
1651                                 chan->log_def.lcsp1,
1652                                 src_info->data_width,
1653                                 dst_info->data_width);
1654
1655         ret = d40_log_sg_to_lli(sg_dst, sg_len,
1656                                 dst_dev_addr,
1657                                 desc->lli_log.dst,
1658                                 chan->log_def.lcsp3,
1659                                 dst_info->data_width,
1660                                 src_info->data_width);
1661
1662         return ret < 0 ? ret : 0;
1663 }
1664
1665 static int
1666 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
1667                 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1668                 unsigned int sg_len, enum dma_data_direction direction,
1669                 dma_addr_t dev_addr)
1670 {
1671         dma_addr_t src_dev_addr = direction == DMA_FROM_DEVICE ? dev_addr : 0;
1672         dma_addr_t dst_dev_addr = direction == DMA_TO_DEVICE ? dev_addr : 0;
1673         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1674         struct stedma40_half_channel_info *src_info = &cfg->src_info;
1675         struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1676         int ret;
1677
1678         ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
1679                                 desc->lli_phy.src,
1680                                 virt_to_phys(desc->lli_phy.src),
1681                                 chan->src_def_cfg,
1682                                 src_info, dst_info);
1683
1684         ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
1685                                 desc->lli_phy.dst,
1686                                 virt_to_phys(desc->lli_phy.dst),
1687                                 chan->dst_def_cfg,
1688                                 dst_info, src_info);
1689
1690         dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
1691                                    desc->lli_pool.size, DMA_TO_DEVICE);
1692
1693         return ret < 0 ? ret : 0;
1694 }
1695
1696
1697 static struct d40_desc *
1698 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1699               unsigned int sg_len, unsigned long dma_flags)
1700 {
1701         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1702         struct d40_desc *desc;
1703         int ret;
1704
1705         desc = d40_desc_get(chan);
1706         if (!desc)
1707                 return NULL;
1708
1709         desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1710                                         cfg->dst_info.data_width);
1711         if (desc->lli_len < 0) {
1712                 chan_err(chan, "Unaligned size\n");
1713                 goto err;
1714         }
1715
1716         ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
1717         if (ret < 0) {
1718                 chan_err(chan, "Could not allocate lli\n");
1719                 goto err;
1720         }
1721
1722
1723         desc->lli_current = 0;
1724         desc->txd.flags = dma_flags;
1725         desc->txd.tx_submit = d40_tx_submit;
1726
1727         dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1728
1729         return desc;
1730
1731 err:
1732         d40_desc_free(chan, desc);
1733         return NULL;
1734 }
1735
1736 static dma_addr_t
1737 d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
1738 {
1739         struct stedma40_platform_data *plat = chan->base->plat_data;
1740         struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1741         dma_addr_t addr;
1742
1743         if (chan->runtime_addr)
1744                 return chan->runtime_addr;
1745
1746         if (direction == DMA_FROM_DEVICE)
1747                 addr = plat->dev_rx[cfg->src_dev_type];
1748         else if (direction == DMA_TO_DEVICE)
1749                 addr = plat->dev_tx[cfg->dst_dev_type];
1750
1751         return addr;
1752 }
1753
1754 static struct dma_async_tx_descriptor *
1755 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
1756             struct scatterlist *sg_dst, unsigned int sg_len,
1757             enum dma_data_direction direction, unsigned long dma_flags)
1758 {
1759         struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
1760         dma_addr_t dev_addr = 0;
1761         struct d40_desc *desc;
1762         unsigned long flags;
1763         int ret;
1764
1765         if (!chan->phy_chan) {
1766                 chan_err(chan, "Cannot prepare unallocated channel\n");
1767                 return NULL;
1768         }
1769
1770         spin_lock_irqsave(&chan->lock, flags);
1771
1772         desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
1773         if (desc == NULL)
1774                 goto err;
1775
1776         if (direction != DMA_NONE)
1777                 dev_addr = d40_get_dev_addr(chan, direction);
1778
1779         if (chan_is_logical(chan))
1780                 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
1781                                       sg_len, direction, dev_addr);
1782         else
1783                 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
1784                                       sg_len, direction, dev_addr);
1785
1786         if (ret) {
1787                 chan_err(chan, "Failed to prepare %s sg job: %d\n",
1788                          chan_is_logical(chan) ? "log" : "phy", ret);
1789                 goto err;
1790         }
1791
1792         spin_unlock_irqrestore(&chan->lock, flags);
1793
1794         return &desc->txd;
1795
1796 err:
1797         if (desc)
1798                 d40_desc_free(chan, desc);
1799         spin_unlock_irqrestore(&chan->lock, flags);
1800         return NULL;
1801 }
1802
1803 bool stedma40_filter(struct dma_chan *chan, void *data)
1804 {
1805         struct stedma40_chan_cfg *info = data;
1806         struct d40_chan *d40c =
1807                 container_of(chan, struct d40_chan, chan);
1808         int err;
1809
1810         if (data) {
1811                 err = d40_validate_conf(d40c, info);
1812                 if (!err)
1813                         d40c->dma_cfg = *info;
1814         } else
1815                 err = d40_config_memcpy(d40c);
1816
1817         if (!err)
1818                 d40c->configured = true;
1819
1820         return err == 0;
1821 }
1822 EXPORT_SYMBOL(stedma40_filter);
1823
1824 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1825 {
1826         bool realtime = d40c->dma_cfg.realtime;
1827         bool highprio = d40c->dma_cfg.high_priority;
1828         u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1829         u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1830         u32 event = D40_TYPE_TO_EVENT(dev_type);
1831         u32 group = D40_TYPE_TO_GROUP(dev_type);
1832         u32 bit = 1 << event;
1833
1834         /* Destination event lines are stored in the upper halfword */
1835         if (!src)
1836                 bit <<= 16;
1837
1838         writel(bit, d40c->base->virtbase + prioreg + group * 4);
1839         writel(bit, d40c->base->virtbase + rtreg + group * 4);
1840 }
1841
1842 static void d40_set_prio_realtime(struct d40_chan *d40c)
1843 {
1844         if (d40c->base->rev < 3)
1845                 return;
1846
1847         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
1848             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1849                 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1850
1851         if ((d40c->dma_cfg.dir ==  STEDMA40_MEM_TO_PERIPH) ||
1852             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1853                 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1854 }
1855
1856 /* DMA ENGINE functions */
1857 static int d40_alloc_chan_resources(struct dma_chan *chan)
1858 {
1859         int err;
1860         unsigned long flags;
1861         struct d40_chan *d40c =
1862                 container_of(chan, struct d40_chan, chan);
1863         bool is_free_phy;
1864         spin_lock_irqsave(&d40c->lock, flags);
1865
1866         d40c->completed = chan->cookie = 1;
1867
1868         /* If no dma configuration is set use default configuration (memcpy) */
1869         if (!d40c->configured) {
1870                 err = d40_config_memcpy(d40c);
1871                 if (err) {
1872                         chan_err(d40c, "Failed to configure memcpy channel\n");
1873                         goto fail;
1874                 }
1875         }
1876         is_free_phy = (d40c->phy_chan == NULL);
1877
1878         err = d40_allocate_channel(d40c);
1879         if (err) {
1880                 chan_err(d40c, "Failed to allocate channel\n");
1881                 goto fail;
1882         }
1883
1884         /* Fill in basic CFG register values */
1885         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1886                     &d40c->dst_def_cfg, chan_is_logical(d40c));
1887
1888         d40_set_prio_realtime(d40c);
1889
1890         if (chan_is_logical(d40c)) {
1891                 d40_log_cfg(&d40c->dma_cfg,
1892                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1893
1894                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1895                         d40c->lcpa = d40c->base->lcpa_base +
1896                           d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1897                 else
1898                         d40c->lcpa = d40c->base->lcpa_base +
1899                           d40c->dma_cfg.dst_dev_type *
1900                           D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1901         }
1902
1903         /*
1904          * Only write channel configuration to the DMA if the physical
1905          * resource is free. In case of multiple logical channels
1906          * on the same physical resource, only the first write is necessary.
1907          */
1908         if (is_free_phy)
1909                 d40_config_write(d40c);
1910 fail:
1911         spin_unlock_irqrestore(&d40c->lock, flags);
1912         return err;
1913 }
1914
1915 static void d40_free_chan_resources(struct dma_chan *chan)
1916 {
1917         struct d40_chan *d40c =
1918                 container_of(chan, struct d40_chan, chan);
1919         int err;
1920         unsigned long flags;
1921
1922         if (d40c->phy_chan == NULL) {
1923                 chan_err(d40c, "Cannot free unallocated channel\n");
1924                 return;
1925         }
1926
1927
1928         spin_lock_irqsave(&d40c->lock, flags);
1929
1930         err = d40_free_dma(d40c);
1931
1932         if (err)
1933                 chan_err(d40c, "Failed to free channel\n");
1934         spin_unlock_irqrestore(&d40c->lock, flags);
1935 }
1936
1937 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1938                                                        dma_addr_t dst,
1939                                                        dma_addr_t src,
1940                                                        size_t size,
1941                                                        unsigned long dma_flags)
1942 {
1943         struct scatterlist dst_sg;
1944         struct scatterlist src_sg;
1945
1946         sg_init_table(&dst_sg, 1);
1947         sg_init_table(&src_sg, 1);
1948
1949         sg_dma_address(&dst_sg) = dst;
1950         sg_dma_address(&src_sg) = src;
1951
1952         sg_dma_len(&dst_sg) = size;
1953         sg_dma_len(&src_sg) = size;
1954
1955         return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
1956 }
1957
1958 static struct dma_async_tx_descriptor *
1959 d40_prep_memcpy_sg(struct dma_chan *chan,
1960                    struct scatterlist *dst_sg, unsigned int dst_nents,
1961                    struct scatterlist *src_sg, unsigned int src_nents,
1962                    unsigned long dma_flags)
1963 {
1964         if (dst_nents != src_nents)
1965                 return NULL;
1966
1967         return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
1968 }
1969
1970 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
1971                                                          struct scatterlist *sgl,
1972                                                          unsigned int sg_len,
1973                                                          enum dma_data_direction direction,
1974                                                          unsigned long dma_flags)
1975 {
1976         if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
1977                 return NULL;
1978
1979         return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
1980 }
1981
1982 static enum dma_status d40_tx_status(struct dma_chan *chan,
1983                                      dma_cookie_t cookie,
1984                                      struct dma_tx_state *txstate)
1985 {
1986         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
1987         dma_cookie_t last_used;
1988         dma_cookie_t last_complete;
1989         int ret;
1990
1991         if (d40c->phy_chan == NULL) {
1992                 chan_err(d40c, "Cannot read status of unallocated channel\n");
1993                 return -EINVAL;
1994         }
1995
1996         last_complete = d40c->completed;
1997         last_used = chan->cookie;
1998
1999         if (d40_is_paused(d40c))
2000                 ret = DMA_PAUSED;
2001         else
2002                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2003
2004         dma_set_tx_state(txstate, last_complete, last_used,
2005                          stedma40_residue(chan));
2006
2007         return ret;
2008 }
2009
2010 static void d40_issue_pending(struct dma_chan *chan)
2011 {
2012         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2013         unsigned long flags;
2014
2015         if (d40c->phy_chan == NULL) {
2016                 chan_err(d40c, "Channel is not allocated!\n");
2017                 return;
2018         }
2019
2020         spin_lock_irqsave(&d40c->lock, flags);
2021
2022         /* Busy means that pending jobs are already being processed */
2023         if (!d40c->busy)
2024                 (void) d40_queue_start(d40c);
2025
2026         spin_unlock_irqrestore(&d40c->lock, flags);
2027 }
2028
2029 /* Runtime reconfiguration extension */
2030 static void d40_set_runtime_config(struct dma_chan *chan,
2031                                struct dma_slave_config *config)
2032 {
2033         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2034         struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2035         enum dma_slave_buswidth config_addr_width;
2036         dma_addr_t config_addr;
2037         u32 config_maxburst;
2038         enum stedma40_periph_data_width addr_width;
2039         int psize;
2040
2041         if (config->direction == DMA_FROM_DEVICE) {
2042                 dma_addr_t dev_addr_rx =
2043                         d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2044
2045                 config_addr = config->src_addr;
2046                 if (dev_addr_rx)
2047                         dev_dbg(d40c->base->dev,
2048                                 "channel has a pre-wired RX address %08x "
2049                                 "overriding with %08x\n",
2050                                 dev_addr_rx, config_addr);
2051                 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2052                         dev_dbg(d40c->base->dev,
2053                                 "channel was not configured for peripheral "
2054                                 "to memory transfer (%d) overriding\n",
2055                                 cfg->dir);
2056                 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2057
2058                 config_addr_width = config->src_addr_width;
2059                 config_maxburst = config->src_maxburst;
2060
2061         } else if (config->direction == DMA_TO_DEVICE) {
2062                 dma_addr_t dev_addr_tx =
2063                         d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2064
2065                 config_addr = config->dst_addr;
2066                 if (dev_addr_tx)
2067                         dev_dbg(d40c->base->dev,
2068                                 "channel has a pre-wired TX address %08x "
2069                                 "overriding with %08x\n",
2070                                 dev_addr_tx, config_addr);
2071                 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2072                         dev_dbg(d40c->base->dev,
2073                                 "channel was not configured for memory "
2074                                 "to peripheral transfer (%d) overriding\n",
2075                                 cfg->dir);
2076                 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2077
2078                 config_addr_width = config->dst_addr_width;
2079                 config_maxburst = config->dst_maxburst;
2080
2081         } else {
2082                 dev_err(d40c->base->dev,
2083                         "unrecognized channel direction %d\n",
2084                         config->direction);
2085                 return;
2086         }
2087
2088         switch (config_addr_width) {
2089         case DMA_SLAVE_BUSWIDTH_1_BYTE:
2090                 addr_width = STEDMA40_BYTE_WIDTH;
2091                 break;
2092         case DMA_SLAVE_BUSWIDTH_2_BYTES:
2093                 addr_width = STEDMA40_HALFWORD_WIDTH;
2094                 break;
2095         case DMA_SLAVE_BUSWIDTH_4_BYTES:
2096                 addr_width = STEDMA40_WORD_WIDTH;
2097                 break;
2098         case DMA_SLAVE_BUSWIDTH_8_BYTES:
2099                 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2100                 break;
2101         default:
2102                 dev_err(d40c->base->dev,
2103                         "illegal peripheral address width "
2104                         "requested (%d)\n",
2105                         config->src_addr_width);
2106                 return;
2107         }
2108
2109         if (chan_is_logical(d40c)) {
2110                 if (config_maxburst >= 16)
2111                         psize = STEDMA40_PSIZE_LOG_16;
2112                 else if (config_maxburst >= 8)
2113                         psize = STEDMA40_PSIZE_LOG_8;
2114                 else if (config_maxburst >= 4)
2115                         psize = STEDMA40_PSIZE_LOG_4;
2116                 else
2117                         psize = STEDMA40_PSIZE_LOG_1;
2118         } else {
2119                 if (config_maxburst >= 16)
2120                         psize = STEDMA40_PSIZE_PHY_16;
2121                 else if (config_maxburst >= 8)
2122                         psize = STEDMA40_PSIZE_PHY_8;
2123                 else if (config_maxburst >= 4)
2124                         psize = STEDMA40_PSIZE_PHY_4;
2125                 else if (config_maxburst >= 2)
2126                         psize = STEDMA40_PSIZE_PHY_2;
2127                 else
2128                         psize = STEDMA40_PSIZE_PHY_1;
2129         }
2130
2131         /* Set up all the endpoint configs */
2132         cfg->src_info.data_width = addr_width;
2133         cfg->src_info.psize = psize;
2134         cfg->src_info.big_endian = false;
2135         cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2136         cfg->dst_info.data_width = addr_width;
2137         cfg->dst_info.psize = psize;
2138         cfg->dst_info.big_endian = false;
2139         cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2140
2141         /* Fill in register values */
2142         if (chan_is_logical(d40c))
2143                 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2144         else
2145                 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2146                             &d40c->dst_def_cfg, false);
2147
2148         /* These settings will take precedence later */
2149         d40c->runtime_addr = config_addr;
2150         d40c->runtime_direction = config->direction;
2151         dev_dbg(d40c->base->dev,
2152                 "configured channel %s for %s, data width %d, "
2153                 "maxburst %d bytes, LE, no flow control\n",
2154                 dma_chan_name(chan),
2155                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2156                 config_addr_width,
2157                 config_maxburst);
2158 }
2159
2160 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2161                        unsigned long arg)
2162 {
2163         unsigned long flags;
2164         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2165
2166         if (d40c->phy_chan == NULL) {
2167                 chan_err(d40c, "Channel is not allocated!\n");
2168                 return -EINVAL;
2169         }
2170
2171         switch (cmd) {
2172         case DMA_TERMINATE_ALL:
2173                 spin_lock_irqsave(&d40c->lock, flags);
2174                 d40_term_all(d40c);
2175                 spin_unlock_irqrestore(&d40c->lock, flags);
2176                 return 0;
2177         case DMA_PAUSE:
2178                 return d40_pause(chan);
2179         case DMA_RESUME:
2180                 return d40_resume(chan);
2181         case DMA_SLAVE_CONFIG:
2182                 d40_set_runtime_config(chan,
2183                         (struct dma_slave_config *) arg);
2184                 return 0;
2185         default:
2186                 break;
2187         }
2188
2189         /* Other commands are unimplemented */
2190         return -ENXIO;
2191 }
2192
2193 /* Initialization functions */
2194
2195 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2196                                  struct d40_chan *chans, int offset,
2197                                  int num_chans)
2198 {
2199         int i = 0;
2200         struct d40_chan *d40c;
2201
2202         INIT_LIST_HEAD(&dma->channels);
2203
2204         for (i = offset; i < offset + num_chans; i++) {
2205                 d40c = &chans[i];
2206                 d40c->base = base;
2207                 d40c->chan.device = dma;
2208
2209                 spin_lock_init(&d40c->lock);
2210
2211                 d40c->log_num = D40_PHY_CHAN;
2212
2213                 INIT_LIST_HEAD(&d40c->active);
2214                 INIT_LIST_HEAD(&d40c->queue);
2215                 INIT_LIST_HEAD(&d40c->client);
2216
2217                 tasklet_init(&d40c->tasklet, dma_tasklet,
2218                              (unsigned long) d40c);
2219
2220                 list_add_tail(&d40c->chan.device_node,
2221                               &dma->channels);
2222         }
2223 }
2224
2225 static int __init d40_dmaengine_init(struct d40_base *base,
2226                                      int num_reserved_chans)
2227 {
2228         int err ;
2229
2230         d40_chan_init(base, &base->dma_slave, base->log_chans,
2231                       0, base->num_log_chans);
2232
2233         dma_cap_zero(base->dma_slave.cap_mask);
2234         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2235
2236         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2237         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2238         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2239         base->dma_slave.device_prep_dma_sg = d40_prep_memcpy_sg;
2240         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2241         base->dma_slave.device_tx_status = d40_tx_status;
2242         base->dma_slave.device_issue_pending = d40_issue_pending;
2243         base->dma_slave.device_control = d40_control;
2244         base->dma_slave.dev = base->dev;
2245
2246         err = dma_async_device_register(&base->dma_slave);
2247
2248         if (err) {
2249                 d40_err(base->dev, "Failed to register slave channels\n");
2250                 goto failure1;
2251         }
2252
2253         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2254                       base->num_log_chans, base->plat_data->memcpy_len);
2255
2256         dma_cap_zero(base->dma_memcpy.cap_mask);
2257         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2258         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2259
2260         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2261         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2262         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2263         base->dma_slave.device_prep_dma_sg = d40_prep_memcpy_sg;
2264         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2265         base->dma_memcpy.device_tx_status = d40_tx_status;
2266         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2267         base->dma_memcpy.device_control = d40_control;
2268         base->dma_memcpy.dev = base->dev;
2269         /*
2270          * This controller can only access address at even
2271          * 32bit boundaries, i.e. 2^2
2272          */
2273         base->dma_memcpy.copy_align = 2;
2274
2275         err = dma_async_device_register(&base->dma_memcpy);
2276
2277         if (err) {
2278                 d40_err(base->dev,
2279                         "Failed to regsiter memcpy only channels\n");
2280                 goto failure2;
2281         }
2282
2283         d40_chan_init(base, &base->dma_both, base->phy_chans,
2284                       0, num_reserved_chans);
2285
2286         dma_cap_zero(base->dma_both.cap_mask);
2287         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2288         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2289         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2290
2291         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2292         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2293         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2294         base->dma_slave.device_prep_dma_sg = d40_prep_memcpy_sg;
2295         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2296         base->dma_both.device_tx_status = d40_tx_status;
2297         base->dma_both.device_issue_pending = d40_issue_pending;
2298         base->dma_both.device_control = d40_control;
2299         base->dma_both.dev = base->dev;
2300         base->dma_both.copy_align = 2;
2301         err = dma_async_device_register(&base->dma_both);
2302
2303         if (err) {
2304                 d40_err(base->dev,
2305                         "Failed to register logical and physical capable channels\n");
2306                 goto failure3;
2307         }
2308         return 0;
2309 failure3:
2310         dma_async_device_unregister(&base->dma_memcpy);
2311 failure2:
2312         dma_async_device_unregister(&base->dma_slave);
2313 failure1:
2314         return err;
2315 }
2316
2317 /* Initialization functions. */
2318
2319 static int __init d40_phy_res_init(struct d40_base *base)
2320 {
2321         int i;
2322         int num_phy_chans_avail = 0;
2323         u32 val[2];
2324         int odd_even_bit = -2;
2325
2326         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2327         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2328
2329         for (i = 0; i < base->num_phy_chans; i++) {
2330                 base->phy_res[i].num = i;
2331                 odd_even_bit += 2 * ((i % 2) == 0);
2332                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2333                         /* Mark security only channels as occupied */
2334                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2335                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2336                 } else {
2337                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2338                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2339                         num_phy_chans_avail++;
2340                 }
2341                 spin_lock_init(&base->phy_res[i].lock);
2342         }
2343
2344         /* Mark disabled channels as occupied */
2345         for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2346                 int chan = base->plat_data->disabled_channels[i];
2347
2348                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2349                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2350                 num_phy_chans_avail--;
2351         }
2352
2353         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2354                  num_phy_chans_avail, base->num_phy_chans);
2355
2356         /* Verify settings extended vs standard */
2357         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2358
2359         for (i = 0; i < base->num_phy_chans; i++) {
2360
2361                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2362                     (val[0] & 0x3) != 1)
2363                         dev_info(base->dev,
2364                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2365                                  __func__, i, val[0] & 0x3);
2366
2367                 val[0] = val[0] >> 2;
2368         }
2369
2370         return num_phy_chans_avail;
2371 }
2372
2373 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2374 {
2375         static const struct d40_reg_val dma_id_regs[] = {
2376                 /* Peripheral Id */
2377                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2378                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2379                 /*
2380                  * D40_DREG_PERIPHID2 Depends on HW revision:
2381                  *  DB8500ed has 0x0008,
2382                  *  ? has 0x0018,
2383                  *  DB8500v1 has 0x0028
2384                  *  DB8500v2 has 0x0038
2385                  */
2386                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2387
2388                 /* PCell Id */
2389                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2390                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2391                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2392                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2393         };
2394         struct stedma40_platform_data *plat_data;
2395         struct clk *clk = NULL;
2396         void __iomem *virtbase = NULL;
2397         struct resource *res = NULL;
2398         struct d40_base *base = NULL;
2399         int num_log_chans = 0;
2400         int num_phy_chans;
2401         int i;
2402         u32 val;
2403         u32 rev;
2404
2405         clk = clk_get(&pdev->dev, NULL);
2406
2407         if (IS_ERR(clk)) {
2408                 d40_err(&pdev->dev, "No matching clock found\n");
2409                 goto failure;
2410         }
2411
2412         clk_enable(clk);
2413
2414         /* Get IO for DMAC base address */
2415         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2416         if (!res)
2417                 goto failure;
2418
2419         if (request_mem_region(res->start, resource_size(res),
2420                                D40_NAME " I/O base") == NULL)
2421                 goto failure;
2422
2423         virtbase = ioremap(res->start, resource_size(res));
2424         if (!virtbase)
2425                 goto failure;
2426
2427         /* HW version check */
2428         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2429                 if (dma_id_regs[i].val !=
2430                     readl(virtbase + dma_id_regs[i].reg)) {
2431                         d40_err(&pdev->dev,
2432                                 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2433                                 dma_id_regs[i].val,
2434                                 dma_id_regs[i].reg,
2435                                 readl(virtbase + dma_id_regs[i].reg));
2436                         goto failure;
2437                 }
2438         }
2439
2440         /* Get silicon revision and designer */
2441         val = readl(virtbase + D40_DREG_PERIPHID2);
2442
2443         if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2444             D40_HW_DESIGNER) {
2445                 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2446                         val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2447                         D40_HW_DESIGNER);
2448                 goto failure;
2449         }
2450
2451         rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2452                 D40_DREG_PERIPHID2_REV_POS;
2453
2454         /* The number of physical channels on this HW */
2455         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2456
2457         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2458                  rev, res->start);
2459
2460         plat_data = pdev->dev.platform_data;
2461
2462         /* Count the number of logical channels in use */
2463         for (i = 0; i < plat_data->dev_len; i++)
2464                 if (plat_data->dev_rx[i] != 0)
2465                         num_log_chans++;
2466
2467         for (i = 0; i < plat_data->dev_len; i++)
2468                 if (plat_data->dev_tx[i] != 0)
2469                         num_log_chans++;
2470
2471         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2472                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2473                        sizeof(struct d40_chan), GFP_KERNEL);
2474
2475         if (base == NULL) {
2476                 d40_err(&pdev->dev, "Out of memory\n");
2477                 goto failure;
2478         }
2479
2480         base->rev = rev;
2481         base->clk = clk;
2482         base->num_phy_chans = num_phy_chans;
2483         base->num_log_chans = num_log_chans;
2484         base->phy_start = res->start;
2485         base->phy_size = resource_size(res);
2486         base->virtbase = virtbase;
2487         base->plat_data = plat_data;
2488         base->dev = &pdev->dev;
2489         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2490         base->log_chans = &base->phy_chans[num_phy_chans];
2491
2492         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2493                                 GFP_KERNEL);
2494         if (!base->phy_res)
2495                 goto failure;
2496
2497         base->lookup_phy_chans = kzalloc(num_phy_chans *
2498                                          sizeof(struct d40_chan *),
2499                                          GFP_KERNEL);
2500         if (!base->lookup_phy_chans)
2501                 goto failure;
2502
2503         if (num_log_chans + plat_data->memcpy_len) {
2504                 /*
2505                  * The max number of logical channels are event lines for all
2506                  * src devices and dst devices
2507                  */
2508                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2509                                                  sizeof(struct d40_chan *),
2510                                                  GFP_KERNEL);
2511                 if (!base->lookup_log_chans)
2512                         goto failure;
2513         }
2514
2515         base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2516                                             sizeof(struct d40_desc *) *
2517                                             D40_LCLA_LINK_PER_EVENT_GRP,
2518                                             GFP_KERNEL);
2519         if (!base->lcla_pool.alloc_map)
2520                 goto failure;
2521
2522         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2523                                             0, SLAB_HWCACHE_ALIGN,
2524                                             NULL);
2525         if (base->desc_slab == NULL)
2526                 goto failure;
2527
2528         return base;
2529
2530 failure:
2531         if (!IS_ERR(clk)) {
2532                 clk_disable(clk);
2533                 clk_put(clk);
2534         }
2535         if (virtbase)
2536                 iounmap(virtbase);
2537         if (res)
2538                 release_mem_region(res->start,
2539                                    resource_size(res));
2540         if (virtbase)
2541                 iounmap(virtbase);
2542
2543         if (base) {
2544                 kfree(base->lcla_pool.alloc_map);
2545                 kfree(base->lookup_log_chans);
2546                 kfree(base->lookup_phy_chans);
2547                 kfree(base->phy_res);
2548                 kfree(base);
2549         }
2550
2551         return NULL;
2552 }
2553
2554 static void __init d40_hw_init(struct d40_base *base)
2555 {
2556
2557         static const struct d40_reg_val dma_init_reg[] = {
2558                 /* Clock every part of the DMA block from start */
2559                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2560
2561                 /* Interrupts on all logical channels */
2562                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2563                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2564                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2565                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2566                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2567                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2568                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2569                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2570                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2571                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2572                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2573                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2574         };
2575         int i;
2576         u32 prmseo[2] = {0, 0};
2577         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2578         u32 pcmis = 0;
2579         u32 pcicr = 0;
2580
2581         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2582                 writel(dma_init_reg[i].val,
2583                        base->virtbase + dma_init_reg[i].reg);
2584
2585         /* Configure all our dma channels to default settings */
2586         for (i = 0; i < base->num_phy_chans; i++) {
2587
2588                 activeo[i % 2] = activeo[i % 2] << 2;
2589
2590                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2591                     == D40_ALLOC_PHY) {
2592                         activeo[i % 2] |= 3;
2593                         continue;
2594                 }
2595
2596                 /* Enable interrupt # */
2597                 pcmis = (pcmis << 1) | 1;
2598
2599                 /* Clear interrupt # */
2600                 pcicr = (pcicr << 1) | 1;
2601
2602                 /* Set channel to physical mode */
2603                 prmseo[i % 2] = prmseo[i % 2] << 2;
2604                 prmseo[i % 2] |= 1;
2605
2606         }
2607
2608         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2609         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2610         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2611         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2612
2613         /* Write which interrupt to enable */
2614         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2615
2616         /* Write which interrupt to clear */
2617         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2618
2619 }
2620
2621 static int __init d40_lcla_allocate(struct d40_base *base)
2622 {
2623         struct d40_lcla_pool *pool = &base->lcla_pool;
2624         unsigned long *page_list;
2625         int i, j;
2626         int ret = 0;
2627
2628         /*
2629          * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2630          * To full fill this hardware requirement without wasting 256 kb
2631          * we allocate pages until we get an aligned one.
2632          */
2633         page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2634                             GFP_KERNEL);
2635
2636         if (!page_list) {
2637                 ret = -ENOMEM;
2638                 goto failure;
2639         }
2640
2641         /* Calculating how many pages that are required */
2642         base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2643
2644         for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2645                 page_list[i] = __get_free_pages(GFP_KERNEL,
2646                                                 base->lcla_pool.pages);
2647                 if (!page_list[i]) {
2648
2649                         d40_err(base->dev, "Failed to allocate %d pages.\n",
2650                                 base->lcla_pool.pages);
2651
2652                         for (j = 0; j < i; j++)
2653                                 free_pages(page_list[j], base->lcla_pool.pages);
2654                         goto failure;
2655                 }
2656
2657                 if ((virt_to_phys((void *)page_list[i]) &
2658                      (LCLA_ALIGNMENT - 1)) == 0)
2659                         break;
2660         }
2661
2662         for (j = 0; j < i; j++)
2663                 free_pages(page_list[j], base->lcla_pool.pages);
2664
2665         if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2666                 base->lcla_pool.base = (void *)page_list[i];
2667         } else {
2668                 /*
2669                  * After many attempts and no succees with finding the correct
2670                  * alignment, try with allocating a big buffer.
2671                  */
2672                 dev_warn(base->dev,
2673                          "[%s] Failed to get %d pages @ 18 bit align.\n",
2674                          __func__, base->lcla_pool.pages);
2675                 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2676                                                          base->num_phy_chans +
2677                                                          LCLA_ALIGNMENT,
2678                                                          GFP_KERNEL);
2679                 if (!base->lcla_pool.base_unaligned) {
2680                         ret = -ENOMEM;
2681                         goto failure;
2682                 }
2683
2684                 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2685                                                  LCLA_ALIGNMENT);
2686         }
2687
2688         pool->dma_addr = dma_map_single(base->dev, pool->base,
2689                                         SZ_1K * base->num_phy_chans,
2690                                         DMA_TO_DEVICE);
2691         if (dma_mapping_error(base->dev, pool->dma_addr)) {
2692                 pool->dma_addr = 0;
2693                 ret = -ENOMEM;
2694                 goto failure;
2695         }
2696
2697         writel(virt_to_phys(base->lcla_pool.base),
2698                base->virtbase + D40_DREG_LCLA);
2699 failure:
2700         kfree(page_list);
2701         return ret;
2702 }
2703
2704 static int __init d40_probe(struct platform_device *pdev)
2705 {
2706         int err;
2707         int ret = -ENOENT;
2708         struct d40_base *base;
2709         struct resource *res = NULL;
2710         int num_reserved_chans;
2711         u32 val;
2712
2713         base = d40_hw_detect_init(pdev);
2714
2715         if (!base)
2716                 goto failure;
2717
2718         num_reserved_chans = d40_phy_res_init(base);
2719
2720         platform_set_drvdata(pdev, base);
2721
2722         spin_lock_init(&base->interrupt_lock);
2723         spin_lock_init(&base->execmd_lock);
2724
2725         /* Get IO for logical channel parameter address */
2726         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2727         if (!res) {
2728                 ret = -ENOENT;
2729                 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
2730                 goto failure;
2731         }
2732         base->lcpa_size = resource_size(res);
2733         base->phy_lcpa = res->start;
2734
2735         if (request_mem_region(res->start, resource_size(res),
2736                                D40_NAME " I/O lcpa") == NULL) {
2737                 ret = -EBUSY;
2738                 d40_err(&pdev->dev,
2739                         "Failed to request LCPA region 0x%x-0x%x\n",
2740                         res->start, res->end);
2741                 goto failure;
2742         }
2743
2744         /* We make use of ESRAM memory for this. */
2745         val = readl(base->virtbase + D40_DREG_LCPA);
2746         if (res->start != val && val != 0) {
2747                 dev_warn(&pdev->dev,
2748                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2749                          __func__, val, res->start);
2750         } else
2751                 writel(res->start, base->virtbase + D40_DREG_LCPA);
2752
2753         base->lcpa_base = ioremap(res->start, resource_size(res));
2754         if (!base->lcpa_base) {
2755                 ret = -ENOMEM;
2756                 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
2757                 goto failure;
2758         }
2759
2760         ret = d40_lcla_allocate(base);
2761         if (ret) {
2762                 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
2763                 goto failure;
2764         }
2765
2766         spin_lock_init(&base->lcla_pool.lock);
2767
2768         base->irq = platform_get_irq(pdev, 0);
2769
2770         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2771         if (ret) {
2772                 d40_err(&pdev->dev, "No IRQ defined\n");
2773                 goto failure;
2774         }
2775
2776         err = d40_dmaengine_init(base, num_reserved_chans);
2777         if (err)
2778                 goto failure;
2779
2780         d40_hw_init(base);
2781
2782         dev_info(base->dev, "initialized\n");
2783         return 0;
2784
2785 failure:
2786         if (base) {
2787                 if (base->desc_slab)
2788                         kmem_cache_destroy(base->desc_slab);
2789                 if (base->virtbase)
2790                         iounmap(base->virtbase);
2791
2792                 if (base->lcla_pool.dma_addr)
2793                         dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2794                                          SZ_1K * base->num_phy_chans,
2795                                          DMA_TO_DEVICE);
2796
2797                 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2798                         free_pages((unsigned long)base->lcla_pool.base,
2799                                    base->lcla_pool.pages);
2800
2801                 kfree(base->lcla_pool.base_unaligned);
2802
2803                 if (base->phy_lcpa)
2804                         release_mem_region(base->phy_lcpa,
2805                                            base->lcpa_size);
2806                 if (base->phy_start)
2807                         release_mem_region(base->phy_start,
2808                                            base->phy_size);
2809                 if (base->clk) {
2810                         clk_disable(base->clk);
2811                         clk_put(base->clk);
2812                 }
2813
2814                 kfree(base->lcla_pool.alloc_map);
2815                 kfree(base->lookup_log_chans);
2816                 kfree(base->lookup_phy_chans);
2817                 kfree(base->phy_res);
2818                 kfree(base);
2819         }
2820
2821         d40_err(&pdev->dev, "probe failed\n");
2822         return ret;
2823 }
2824
2825 static struct platform_driver d40_driver = {
2826         .driver = {
2827                 .owner = THIS_MODULE,
2828                 .name  = D40_NAME,
2829         },
2830 };
2831
2832 static int __init stedma40_init(void)
2833 {
2834         return platform_driver_probe(&d40_driver, d40_probe);
2835 }
2836 arch_initcall(stedma40_init);