drivers/dma/img-mdc-dma.c

   1 /*
   2  * IMG Multi-threaded DMA Controller (MDC)
   3  *
   4  * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
   5  * Copyright (C) 2014 Google, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2, as published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/dmapool.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/io.h>
  18 #include <linux/irq.h>
  19 #include <linux/kernel.h>
  20 #include <linux/mfd/syscon.h>
  21 #include <linux/module.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_dma.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/regmap.h>
  27 #include <linux/slab.h>
  28 #include <linux/spinlock.h>
  29
  30 #include "dmaengine.h"
  31 #include "virt-dma.h"
  32
  33 #define MDC_MAX_DMA_CHANNELS                    32
  34
  35 #define MDC_GENERAL_CONFIG                      0x000
  36 #define MDC_GENERAL_CONFIG_LIST_IEN             BIT(31)
  37 #define MDC_GENERAL_CONFIG_IEN                  BIT(29)
  38 #define MDC_GENERAL_CONFIG_LEVEL_INT            BIT(28)
  39 #define MDC_GENERAL_CONFIG_INC_W                BIT(12)
  40 #define MDC_GENERAL_CONFIG_INC_R                BIT(8)
  41 #define MDC_GENERAL_CONFIG_PHYSICAL_W           BIT(7)
  42 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT        4
  43 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK         0x7
  44 #define MDC_GENERAL_CONFIG_PHYSICAL_R           BIT(3)
  45 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT        0
  46 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK         0x7
  47
  48 #define MDC_READ_PORT_CONFIG                    0x004
  49 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT      28
  50 #define MDC_READ_PORT_CONFIG_STHREAD_MASK       0xf
  51 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT      24
  52 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK       0xf
  53 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT      16
  54 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK       0xf
  55 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT   4
  56 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK    0xff
  57 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE        BIT(1)
  58
  59 #define MDC_READ_ADDRESS                        0x008
  60
  61 #define MDC_WRITE_ADDRESS                       0x00c
  62
  63 #define MDC_TRANSFER_SIZE                       0x010
  64 #define MDC_TRANSFER_SIZE_MASK                  0xffffff
  65
  66 #define MDC_LIST_NODE_ADDRESS                   0x014
  67
  68 #define MDC_CMDS_PROCESSED                      0x018
  69 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
  70 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK  0x3f
  71 #define MDC_CMDS_PROCESSED_INT_ACTIVE           BIT(8)
  72 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT      0
  73 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK       0x3f
  74
  75 #define MDC_CONTROL_AND_STATUS                  0x01c
  76 #define MDC_CONTROL_AND_STATUS_CANCEL           BIT(20)
  77 #define MDC_CONTROL_AND_STATUS_LIST_EN          BIT(4)
  78 #define MDC_CONTROL_AND_STATUS_EN               BIT(0)
  79
  80 #define MDC_ACTIVE_TRANSFER_SIZE                0x030
  81
  82 #define MDC_GLOBAL_CONFIG_A                             0x900
  83 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT       16
  84 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK        0xff
  85 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT          8
  86 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK           0xff
  87 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT         0
  88 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK          0xff
  89
  90 struct mdc_hw_list_desc {
  91         u32 gen_conf;
  92         u32 readport_conf;
  93         u32 read_addr;
  94         u32 write_addr;
  95         u32 xfer_size;
  96         u32 node_addr;
  97         u32 cmds_done;
  98         u32 ctrl_status;
  99         /*
 100          * Not part of the list descriptor, but instead used by the CPU to
 101          * traverse the list.
 102          */
 103         struct mdc_hw_list_desc *next_desc;
 104 };
 105
 106 struct mdc_tx_desc {
 107         struct mdc_chan *chan;
 108         struct virt_dma_desc vd;
 109         dma_addr_t list_phys;
 110         struct mdc_hw_list_desc *list;
 111         bool cyclic;
 112         bool cmd_loaded;
 113         unsigned int list_len;
 114         unsigned int list_period_len;
 115         size_t list_xfer_size;
 116         unsigned int list_cmds_done;
 117 };
 118
 119 struct mdc_chan {
 120         struct mdc_dma *mdma;
 121         struct virt_dma_chan vc;
 122         struct dma_slave_config config;
 123         struct mdc_tx_desc *desc;
 124         int irq;
 125         unsigned int periph;
 126         unsigned int thread;
 127         unsigned int chan_nr;
 128 };
 129
 130 struct mdc_dma_soc_data {
 131         void (*enable_chan)(struct mdc_chan *mchan);
 132         void (*disable_chan)(struct mdc_chan *mchan);
 133 };
 134
 135 struct mdc_dma {
 136         struct dma_device dma_dev;
 137         void __iomem *regs;
 138         struct clk *clk;
 139         struct dma_pool *desc_pool;
 140         struct regmap *periph_regs;
 141         spinlock_t lock;
 142         unsigned int nr_threads;
 143         unsigned int nr_channels;
 144         unsigned int bus_width;
 145         unsigned int max_burst_mult;
 146         unsigned int max_xfer_size;
 147         const struct mdc_dma_soc_data *soc;
 148         struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
 149 };
 150
 151 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
 152 {
 153         return readl(mdma->regs + reg);
 154 }
 155
 156 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
 157 {
 158         writel(val, mdma->regs + reg);
 159 }
 160
 161 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
 162 {
 163         return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
 164 }
 165
 166 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
 167 {
 168         mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
 169 }
 170
 171 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
 172 {
 173         return container_of(to_virt_chan(c), struct mdc_chan, vc);
 174 }
 175
 176 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
 177 {
 178         struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
 179
 180         return container_of(vdesc, struct mdc_tx_desc, vd);
 181 }
 182
 183 static inline struct device *mdma2dev(struct mdc_dma *mdma)
 184 {
 185         return mdma->dma_dev.dev;
 186 }
 187
 188 static inline unsigned int to_mdc_width(unsigned int bytes)
 189 {
 190         return ffs(bytes) - 1;
 191 }
 192
 193 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
 194                                       unsigned int bytes)
 195 {
 196         ldesc->gen_conf |= to_mdc_width(bytes) <<
 197                 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
 198 }
 199
 200 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
 201                                        unsigned int bytes)
 202 {
 203         ldesc->gen_conf |= to_mdc_width(bytes) <<
 204                 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
 205 }
 206
 207 static void mdc_list_desc_config(struct mdc_chan *mchan,
 208                                  struct mdc_hw_list_desc *ldesc,
 209                                  enum dma_transfer_direction dir,
 210                                  dma_addr_t src, dma_addr_t dst, size_t len)
 211 {
 212         struct mdc_dma *mdma = mchan->mdma;
 213         unsigned int max_burst, burst_size;
 214
 215         ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
 216                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 217                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 218         ldesc->readport_conf =
 219                 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 220                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 221                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 222         ldesc->read_addr = src;
 223         ldesc->write_addr = dst;
 224         ldesc->xfer_size = len - 1;
 225         ldesc->node_addr = 0;
 226         ldesc->cmds_done = 0;
 227         ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
 228                 MDC_CONTROL_AND_STATUS_EN;
 229         ldesc->next_desc = NULL;
 230
 231         if (IS_ALIGNED(dst, mdma->bus_width) &&
 232             IS_ALIGNED(src, mdma->bus_width))
 233                 max_burst = mdma->bus_width * mdma->max_burst_mult;
 234         else
 235                 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
 236
 237         if (dir == DMA_MEM_TO_DEV) {
 238                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
 239                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 240                 mdc_set_read_width(ldesc, mdma->bus_width);
 241                 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
 242                 burst_size = min(max_burst, mchan->config.dst_maxburst *
 243                                  mchan->config.dst_addr_width);
 244         } else if (dir == DMA_DEV_TO_MEM) {
 245                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
 246                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 247                 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
 248                 mdc_set_write_width(ldesc, mdma->bus_width);
 249                 burst_size = min(max_burst, mchan->config.src_maxburst *
 250                                  mchan->config.src_addr_width);
 251         } else {
 252                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
 253                         MDC_GENERAL_CONFIG_INC_W;
 254                 mdc_set_read_width(ldesc, mdma->bus_width);
 255                 mdc_set_write_width(ldesc, mdma->bus_width);
 256                 burst_size = max_burst;
 257         }
 258         ldesc->readport_conf |= (burst_size - 1) <<
 259                 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
 260 }
 261
 262 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
 263 {
 264         struct mdc_dma *mdma = mdesc->chan->mdma;
 265         struct mdc_hw_list_desc *curr, *next;
 266         dma_addr_t curr_phys, next_phys;
 267
 268         curr = mdesc->list;
 269         curr_phys = mdesc->list_phys;
 270         while (curr) {
 271                 next = curr->next_desc;
 272                 next_phys = curr->node_addr;
 273                 dma_pool_free(mdma->desc_pool, curr, curr_phys);
 274                 curr = next;
 275                 curr_phys = next_phys;
 276         }
 277 }
 278
 279 static void mdc_desc_free(struct virt_dma_desc *vd)
 280 {
 281         struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
 282
 283         mdc_list_desc_free(mdesc);
 284         kfree(mdesc);
 285 }
 286
 287 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 288         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
 289         unsigned long flags)
 290 {
 291         struct mdc_chan *mchan = to_mdc_chan(chan);
 292         struct mdc_dma *mdma = mchan->mdma;
 293         struct mdc_tx_desc *mdesc;
 294         struct mdc_hw_list_desc *curr, *prev = NULL;
 295         dma_addr_t curr_phys;
 296
 297         if (!len)
 298                 return NULL;
 299
 300         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 301         if (!mdesc)
 302                 return NULL;
 303         mdesc->chan = mchan;
 304         mdesc->list_xfer_size = len;
 305
 306         while (len > 0) {
 307                 size_t xfer_size;
 308
 309                 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
 310                 if (!curr)
 311                         goto free_desc;
 312
 313                 if (prev) {
 314                         prev->node_addr = curr_phys;
 315                         prev->next_desc = curr;
 316                 } else {
 317                         mdesc->list_phys = curr_phys;
 318                         mdesc->list = curr;
 319                 }
 320
 321                 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
 322
 323                 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
 324                                      xfer_size);
 325
 326                 prev = curr;
 327
 328                 mdesc->list_len++;
 329                 src += xfer_size;
 330                 dest += xfer_size;
 331                 len -= xfer_size;
 332         }
 333
 334         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 335
 336 free_desc:
 337         mdc_desc_free(&mdesc->vd);
 338
 339         return NULL;
 340 }
 341
 342 static int mdc_check_slave_width(struct mdc_chan *mchan,
 343                                  enum dma_transfer_direction dir)
 344 {
 345         enum dma_slave_buswidth width;
 346
 347         if (dir == DMA_MEM_TO_DEV)
 348                 width = mchan->config.dst_addr_width;
 349         else
 350                 width = mchan->config.src_addr_width;
 351
 352         switch (width) {
 353         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 354         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 355         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 356         case DMA_SLAVE_BUSWIDTH_8_BYTES:
 357                 break;
 358         default:
 359                 return -EINVAL;
 360         }
 361
 362         if (width > mchan->mdma->bus_width)
 363                 return -EINVAL;
 364
 365         return 0;
 366 }
 367
 368 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 369         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 370         size_t period_len, enum dma_transfer_direction dir,
 371         unsigned long flags)
 372 {
 373         struct mdc_chan *mchan = to_mdc_chan(chan);
 374         struct mdc_dma *mdma = mchan->mdma;
 375         struct mdc_tx_desc *mdesc;
 376         struct mdc_hw_list_desc *curr, *prev = NULL;
 377         dma_addr_t curr_phys;
 378
 379         if (!buf_len && !period_len)
 380                 return NULL;
 381
 382         if (!is_slave_direction(dir))
 383                 return NULL;
 384
 385         if (mdc_check_slave_width(mchan, dir) < 0)
 386                 return NULL;
 387
 388         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 389         if (!mdesc)
 390                 return NULL;
 391         mdesc->chan = mchan;
 392         mdesc->cyclic = true;
 393         mdesc->list_xfer_size = buf_len;
 394         mdesc->list_period_len = DIV_ROUND_UP(period_len,
 395                                               mdma->max_xfer_size);
 396
 397         while (buf_len > 0) {
 398                 size_t remainder = min(period_len, buf_len);
 399
 400                 while (remainder > 0) {
 401                         size_t xfer_size;
 402
 403                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 404                                               &curr_phys);
 405                         if (!curr)
 406                                 goto free_desc;
 407
 408                         if (!prev) {
 409                                 mdesc->list_phys = curr_phys;
 410                                 mdesc->list = curr;
 411                         } else {
 412                                 prev->node_addr = curr_phys;
 413                                 prev->next_desc = curr;
 414                         }
 415
 416                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 417                                           remainder);
 418
 419                         if (dir == DMA_MEM_TO_DEV) {
 420                                 mdc_list_desc_config(mchan, curr, dir,
 421                                                      buf_addr,
 422                                                      mchan->config.dst_addr,
 423                                                      xfer_size);
 424                         } else {
 425                                 mdc_list_desc_config(mchan, curr, dir,
 426                                                      mchan->config.src_addr,
 427                                                      buf_addr,
 428                                                      xfer_size);
 429                         }
 430
 431                         prev = curr;
 432
 433                         mdesc->list_len++;
 434                         buf_addr += xfer_size;
 435                         buf_len -= xfer_size;
 436                         remainder -= xfer_size;
 437                 }
 438         }
 439         prev->node_addr = mdesc->list_phys;
 440
 441         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 442
 443 free_desc:
 444         mdc_desc_free(&mdesc->vd);
 445
 446         return NULL;
 447 }
 448
 449 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 450         struct dma_chan *chan, struct scatterlist *sgl,
 451         unsigned int sg_len, enum dma_transfer_direction dir,
 452         unsigned long flags, void *context)
 453 {
 454         struct mdc_chan *mchan = to_mdc_chan(chan);
 455         struct mdc_dma *mdma = mchan->mdma;
 456         struct mdc_tx_desc *mdesc;
 457         struct scatterlist *sg;
 458         struct mdc_hw_list_desc *curr, *prev = NULL;
 459         dma_addr_t curr_phys;
 460         unsigned int i;
 461
 462         if (!sgl)
 463                 return NULL;
 464
 465         if (!is_slave_direction(dir))
 466                 return NULL;
 467
 468         if (mdc_check_slave_width(mchan, dir) < 0)
 469                 return NULL;
 470
 471         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 472         if (!mdesc)
 473                 return NULL;
 474         mdesc->chan = mchan;
 475
 476         for_each_sg(sgl, sg, sg_len, i) {
 477                 dma_addr_t buf = sg_dma_address(sg);
 478                 size_t buf_len = sg_dma_len(sg);
 479
 480                 while (buf_len > 0) {
 481                         size_t xfer_size;
 482
 483                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 484                                               &curr_phys);
 485                         if (!curr)
 486                                 goto free_desc;
 487
 488                         if (!prev) {
 489                                 mdesc->list_phys = curr_phys;
 490                                 mdesc->list = curr;
 491                         } else {
 492                                 prev->node_addr = curr_phys;
 493                                 prev->next_desc = curr;
 494                         }
 495
 496                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 497                                           buf_len);
 498
 499                         if (dir == DMA_MEM_TO_DEV) {
 500                                 mdc_list_desc_config(mchan, curr, dir, buf,
 501                                                      mchan->config.dst_addr,
 502                                                      xfer_size);
 503                         } else {
 504                                 mdc_list_desc_config(mchan, curr, dir,
 505                                                      mchan->config.src_addr,
 506                                                      buf, xfer_size);
 507                         }
 508
 509                         prev = curr;
 510
 511                         mdesc->list_len++;
 512                         mdesc->list_xfer_size += xfer_size;
 513                         buf += xfer_size;
 514                         buf_len -= xfer_size;
 515                 }
 516         }
 517
 518         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 519
 520 free_desc:
 521         mdc_desc_free(&mdesc->vd);
 522
 523         return NULL;
 524 }
 525
 526 static void mdc_issue_desc(struct mdc_chan *mchan)
 527 {
 528         struct mdc_dma *mdma = mchan->mdma;
 529         struct virt_dma_desc *vd;
 530         struct mdc_tx_desc *mdesc;
 531         u32 val;
 532
 533         vd = vchan_next_desc(&mchan->vc);
 534         if (!vd)
 535                 return;
 536
 537         list_del(&vd->node);
 538
 539         mdesc = to_mdc_desc(&vd->tx);
 540         mchan->desc = mdesc;
 541
 542         dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
 543                 mchan->chan_nr);
 544
 545         mdma->soc->enable_chan(mchan);
 546
 547         val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
 548         val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
 549                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 550                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 551         mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
 552         val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 553                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 554                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 555         mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
 556         mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
 557         val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
 558         val |= MDC_CONTROL_AND_STATUS_LIST_EN;
 559         mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
 560 }
 561
 562 static void mdc_issue_pending(struct dma_chan *chan)
 563 {
 564         struct mdc_chan *mchan = to_mdc_chan(chan);
 565         unsigned long flags;
 566
 567         spin_lock_irqsave(&mchan->vc.lock, flags);
 568         if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
 569                 mdc_issue_desc(mchan);
 570         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 571 }
 572
 573 static enum dma_status mdc_tx_status(struct dma_chan *chan,
 574         dma_cookie_t cookie, struct dma_tx_state *txstate)
 575 {
 576         struct mdc_chan *mchan = to_mdc_chan(chan);
 577         struct mdc_tx_desc *mdesc;
 578         struct virt_dma_desc *vd;
 579         unsigned long flags;
 580         size_t bytes = 0;
 581         int ret;
 582
 583         ret = dma_cookie_status(chan, cookie, txstate);
 584         if (ret == DMA_COMPLETE)
 585                 return ret;
 586
 587         if (!txstate)
 588                 return ret;
 589
 590         spin_lock_irqsave(&mchan->vc.lock, flags);
 591         vd = vchan_find_desc(&mchan->vc, cookie);
 592         if (vd) {
 593                 mdesc = to_mdc_desc(&vd->tx);
 594                 bytes = mdesc->list_xfer_size;
 595         } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
 596                 struct mdc_hw_list_desc *ldesc;
 597                 u32 val1, val2, done, processed, residue;
 598                 int i, cmds;
 599
 600                 mdesc = mchan->desc;
 601
 602                 /*
 603                  * Determine the number of commands that haven't been
 604                  * processed (handled by the IRQ handler) yet.
 605                  */
 606                 do {
 607                         val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 608                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 609                         residue = mdc_chan_readl(mchan,
 610                                                  MDC_ACTIVE_TRANSFER_SIZE);
 611                         val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 612                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 613                 } while (val1 != val2);
 614
 615                 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 616                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 617                 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 618                         MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 619                 cmds = (done - processed) %
 620                         (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
 621
 622                 /*
 623                  * If the command loaded event hasn't been processed yet, then
 624                  * the difference above includes an extra command.
 625                  */
 626                 if (!mdesc->cmd_loaded)
 627                         cmds--;
 628                 else
 629                         cmds += mdesc->list_cmds_done;
 630
 631                 bytes = mdesc->list_xfer_size;
 632                 ldesc = mdesc->list;
 633                 for (i = 0; i < cmds; i++) {
 634                         bytes -= ldesc->xfer_size + 1;
 635                         ldesc = ldesc->next_desc;
 636                 }
 637                 if (ldesc) {
 638                         if (residue != MDC_TRANSFER_SIZE_MASK)
 639                                 bytes -= ldesc->xfer_size - residue;
 640                         else
 641                                 bytes -= ldesc->xfer_size + 1;
 642                 }
 643         }
 644         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 645
 646         dma_set_residue(txstate, bytes);
 647
 648         return ret;
 649 }
 650
 651 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
 652 {
 653         u32 val, processed, done1, done2;
 654         unsigned int ret;
 655
 656         val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 657         processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 658                                 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 659         /*
 660          * CMDS_DONE may have incremented between reading CMDS_PROCESSED
 661          * and clearing INT_ACTIVE.  Re-read CMDS_PROCESSED to ensure we
 662          * didn't miss a command completion.
 663          */
 664         do {
 665                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 666
 667                 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 668                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 669
 670                 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
 671                           MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
 672                          MDC_CMDS_PROCESSED_INT_ACTIVE);
 673
 674                 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
 675
 676                 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
 677
 678                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 679
 680                 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 681                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 682         } while (done1 != done2);
 683
 684         if (done1 >= processed)
 685                 ret = done1 - processed;
 686         else
 687                 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
 688                         processed) + done1;
 689
 690         return ret;
 691 }
 692
 693 static int mdc_terminate_all(struct dma_chan *chan)
 694 {
 695         struct mdc_chan *mchan = to_mdc_chan(chan);
 696         struct mdc_tx_desc *mdesc;
 697         unsigned long flags;
 698         LIST_HEAD(head);
 699
 700         spin_lock_irqsave(&mchan->vc.lock, flags);
 701
 702         mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
 703                         MDC_CONTROL_AND_STATUS);
 704
 705         mdesc = mchan->desc;
 706         mchan->desc = NULL;
 707         vchan_get_all_descriptors(&mchan->vc, &head);
 708
 709         mdc_get_new_events(mchan);
 710
 711         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 712
 713         if (mdesc)
 714                 mdc_desc_free(&mdesc->vd);
 715         vchan_dma_desc_free_list(&mchan->vc, &head);
 716
 717         return 0;
 718 }
 719
 720 static int mdc_slave_config(struct dma_chan *chan,
 721                             struct dma_slave_config *config)
 722 {
 723         struct mdc_chan *mchan = to_mdc_chan(chan);
 724         unsigned long flags;
 725
 726         spin_lock_irqsave(&mchan->vc.lock, flags);
 727         mchan->config = *config;
 728         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 729
 730         return 0;
 731 }
 732
 733 static void mdc_free_chan_resources(struct dma_chan *chan)
 734 {
 735         struct mdc_chan *mchan = to_mdc_chan(chan);
 736         struct mdc_dma *mdma = mchan->mdma;
 737
 738         mdc_terminate_all(chan);
 739
 740         mdma->soc->disable_chan(mchan);
 741 }
 742
 743 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
 744 {
 745         struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
 746         struct mdc_tx_desc *mdesc;
 747         unsigned int i, new_events;
 748
 749         spin_lock(&mchan->vc.lock);
 750
 751         dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
 752
 753         new_events = mdc_get_new_events(mchan);
 754
 755         if (!new_events)
 756                 goto out;
 757
 758         mdesc = mchan->desc;
 759         if (!mdesc) {
 760                 dev_warn(mdma2dev(mchan->mdma),
 761                          "IRQ with no active descriptor on channel %d\n",
 762                          mchan->chan_nr);
 763                 goto out;
 764         }
 765
 766         for (i = 0; i < new_events; i++) {
 767                 /*
 768                  * The first interrupt in a transfer indicates that the
 769                  * command list has been loaded, not that a command has
 770                  * been completed.
 771                  */
 772                 if (!mdesc->cmd_loaded) {
 773                         mdesc->cmd_loaded = true;
 774                         continue;
 775                 }
 776
 777                 mdesc->list_cmds_done++;
 778                 if (mdesc->cyclic) {
 779                         mdesc->list_cmds_done %= mdesc->list_len;
 780                         if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
 781                                 vchan_cyclic_callback(&mdesc->vd);
 782                 } else if (mdesc->list_cmds_done == mdesc->list_len) {
 783                         mchan->desc = NULL;
 784                         vchan_cookie_complete(&mdesc->vd);
 785                         mdc_issue_desc(mchan);
 786                         break;
 787                 }
 788         }
 789 out:
 790         spin_unlock(&mchan->vc.lock);
 791
 792         return IRQ_HANDLED;
 793 }
 794
 795 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
 796                                      struct of_dma *ofdma)
 797 {
 798         struct mdc_dma *mdma = ofdma->of_dma_data;
 799         struct dma_chan *chan;
 800
 801         if (dma_spec->args_count != 3)
 802                 return NULL;
 803
 804         list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
 805                 struct mdc_chan *mchan = to_mdc_chan(chan);
 806
 807                 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
 808                         continue;
 809                 if (dma_get_slave_channel(chan)) {
 810                         mchan->periph = dma_spec->args[0];
 811                         mchan->thread = dma_spec->args[2];
 812                         return chan;
 813                 }
 814         }
 815
 816         return NULL;
 817 }
 818
 819 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch)       (0x120 + 0x4 * ((ch) / 4))
 820 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
 821 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK      0x3f
 822
 823 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
 824 {
 825         struct mdc_dma *mdma = mchan->mdma;
 826
 827         regmap_update_bits(mdma->periph_regs,
 828                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 829                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 830                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 831                            mchan->periph <<
 832                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
 833 }
 834
 835 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
 836 {
 837         struct mdc_dma *mdma = mchan->mdma;
 838
 839         regmap_update_bits(mdma->periph_regs,
 840                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 841                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 842                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 843                            0);
 844 }
 845
 846 static const struct mdc_dma_soc_data pistachio_mdc_data = {
 847         .enable_chan = pistachio_mdc_enable_chan,
 848         .disable_chan = pistachio_mdc_disable_chan,
 849 };
 850
 851 static const struct of_device_id mdc_dma_of_match[] = {
 852         { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
 853         { },
 854 };
 855 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
 856
 857 static int mdc_dma_probe(struct platform_device *pdev)
 858 {
 859         struct mdc_dma *mdma;
 860         struct resource *res;
 861         unsigned int i;
 862         u32 val;
 863         int ret;
 864
 865         mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
 866         if (!mdma)
 867                 return -ENOMEM;
 868         platform_set_drvdata(pdev, mdma);
 869
 870         mdma->soc = of_device_get_match_data(&pdev->dev);
 871
 872         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 873         mdma->regs = devm_ioremap_resource(&pdev->dev, res);
 874         if (IS_ERR(mdma->regs))
 875                 return PTR_ERR(mdma->regs);
 876
 877         mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 878                                                             "img,cr-periph");
 879         if (IS_ERR(mdma->periph_regs))
 880                 return PTR_ERR(mdma->periph_regs);
 881
 882         mdma->clk = devm_clk_get(&pdev->dev, "sys");
 883         if (IS_ERR(mdma->clk))
 884                 return PTR_ERR(mdma->clk);
 885
 886         ret = clk_prepare_enable(mdma->clk);
 887         if (ret)
 888                 return ret;
 889
 890         dma_cap_zero(mdma->dma_dev.cap_mask);
 891         dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
 892         dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
 893         dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
 894         dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
 895
 896         val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
 897         mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
 898                 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
 899         mdma->nr_threads =
 900                 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
 901                       MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
 902         mdma->bus_width =
 903                 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
 904                        MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
 905         /*
 906          * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
 907          * are supported, this makes it possible for the value reported in
 908          * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
 909          * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
 910          * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining.  To eliminate this
 911          * ambiguity, restrict transfer sizes to one bus-width less than the
 912          * actual maximum.
 913          */
 914         mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
 915
 916         of_property_read_u32(pdev->dev.of_node, "dma-channels",
 917                              &mdma->nr_channels);
 918         ret = of_property_read_u32(pdev->dev.of_node,
 919                                    "img,max-burst-multiplier",
 920                                    &mdma->max_burst_mult);
 921         if (ret)
 922                 goto disable_clk;
 923
 924         mdma->dma_dev.dev = &pdev->dev;
 925         mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
 926         mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
 927         mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
 928         mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
 929         mdma->dma_dev.device_tx_status = mdc_tx_status;
 930         mdma->dma_dev.device_issue_pending = mdc_issue_pending;
 931         mdma->dma_dev.device_terminate_all = mdc_terminate_all;
 932         mdma->dma_dev.device_config = mdc_slave_config;
 933
 934         mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 935         mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 936         for (i = 1; i <= mdma->bus_width; i <<= 1) {
 937                 mdma->dma_dev.src_addr_widths |= BIT(i);
 938                 mdma->dma_dev.dst_addr_widths |= BIT(i);
 939         }
 940
 941         INIT_LIST_HEAD(&mdma->dma_dev.channels);
 942         for (i = 0; i < mdma->nr_channels; i++) {
 943                 struct mdc_chan *mchan = &mdma->channels[i];
 944
 945                 mchan->mdma = mdma;
 946                 mchan->chan_nr = i;
 947                 mchan->irq = platform_get_irq(pdev, i);
 948                 if (mchan->irq < 0) {
 949                         ret = mchan->irq;
 950                         goto disable_clk;
 951                 }
 952                 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
 953                                        IRQ_TYPE_LEVEL_HIGH,
 954                                        dev_name(&pdev->dev), mchan);
 955                 if (ret < 0)
 956                         goto disable_clk;
 957
 958                 mchan->vc.desc_free = mdc_desc_free;
 959                 vchan_init(&mchan->vc, &mdma->dma_dev);
 960         }
 961
 962         mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
 963                                            sizeof(struct mdc_hw_list_desc),
 964                                            4, 0);
 965         if (!mdma->desc_pool) {
 966                 ret = -ENOMEM;
 967                 goto disable_clk;
 968         }
 969
 970         ret = dma_async_device_register(&mdma->dma_dev);
 971         if (ret)
 972                 goto disable_clk;
 973
 974         ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
 975         if (ret)
 976                 goto unregister;
 977
 978         dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
 979                  mdma->nr_channels, mdma->nr_threads);
 980
 981         return 0;
 982
 983 unregister:
 984         dma_async_device_unregister(&mdma->dma_dev);
 985 disable_clk:
 986         clk_disable_unprepare(mdma->clk);
 987         return ret;
 988 }
 989
 990 static int mdc_dma_remove(struct platform_device *pdev)
 991 {
 992         struct mdc_dma *mdma = platform_get_drvdata(pdev);
 993         struct mdc_chan *mchan, *next;
 994
 995         of_dma_controller_free(pdev->dev.of_node);
 996         dma_async_device_unregister(&mdma->dma_dev);
 997
 998         list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
 999                                  vc.chan.device_node) {
1000                 list_del(&mchan->vc.chan.device_node);
1001
1002                 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1003
1004                 tasklet_kill(&mchan->vc.task);
1005         }
1006
1007         clk_disable_unprepare(mdma->clk);
1008
1009         return 0;
1010 }
1011
1012 static struct platform_driver mdc_dma_driver = {
1013         .driver = {
1014                 .name = "img-mdc-dma",
1015                 .of_match_table = of_match_ptr(mdc_dma_of_match),
1016         },
1017         .probe = mdc_dma_probe,
1018         .remove = mdc_dma_remove,
1019 };
1020 module_platform_driver(mdc_dma_driver);
1021
1022 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1023 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1024 MODULE_LICENSE("GPL v2");