drivers/dma/ti/k3-udma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   4  *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/delay.h>
  10 #include <linux/dmaengine.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/err.h>
  14 #include <linux/init.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/list.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/slab.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/sys_soc.h>
  21 #include <linux/of.h>
  22 #include <linux/of_dma.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_irq.h>
  25 #include <linux/workqueue.h>
  26 #include <linux/completion.h>
  27 #include <linux/soc/ti/k3-ringacc.h>
  28 #include <linux/soc/ti/ti_sci_protocol.h>
  29 #include <linux/soc/ti/ti_sci_inta_msi.h>
  30 #include <linux/dma/k3-event-router.h>
  31 #include <linux/dma/ti-cppi5.h>
  32
  33 #include "../virt-dma.h"
  34 #include "k3-udma.h"
  35 #include "k3-psil-priv.h"
  36
  37 struct udma_static_tr {
  38         u8 elsize; /* RPSTR0 */
  39         u16 elcnt; /* RPSTR0 */
  40         u16 bstcnt; /* RPSTR1 */
  41 };
  42
  43 #define K3_UDMA_MAX_RFLOWS              1024
  44 #define K3_UDMA_DEFAULT_RING_SIZE       16
  45
  46 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */
  47 #define UDMA_RFLOW_SRCTAG_NONE          0
  48 #define UDMA_RFLOW_SRCTAG_CFG_TAG       1
  49 #define UDMA_RFLOW_SRCTAG_FLOW_ID       2
  50 #define UDMA_RFLOW_SRCTAG_SRC_TAG       4
  51
  52 #define UDMA_RFLOW_DSTTAG_NONE          0
  53 #define UDMA_RFLOW_DSTTAG_CFG_TAG       1
  54 #define UDMA_RFLOW_DSTTAG_FLOW_ID       2
  55 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO    4
  56 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI    5
  57
  58 struct udma_chan;
  59
  60 enum k3_dma_type {
  61         DMA_TYPE_UDMA = 0,
  62         DMA_TYPE_BCDMA,
  63         DMA_TYPE_PKTDMA,
  64 };
  65
  66 enum udma_mmr {
  67         MMR_GCFG = 0,
  68         MMR_BCHANRT,
  69         MMR_RCHANRT,
  70         MMR_TCHANRT,
  71         MMR_LAST,
  72 };
  73
  74 static const char * const mmr_names[] = {
  75         [MMR_GCFG] = "gcfg",
  76         [MMR_BCHANRT] = "bchanrt",
  77         [MMR_RCHANRT] = "rchanrt",
  78         [MMR_TCHANRT] = "tchanrt",
  79 };
  80
  81 struct udma_tchan {
  82         void __iomem *reg_rt;
  83
  84         int id;
  85         struct k3_ring *t_ring; /* Transmit ring */
  86         struct k3_ring *tc_ring; /* Transmit Completion ring */
  87         int tflow_id; /* applicable only for PKTDMA */
  88
  89 };
  90
  91 #define udma_bchan udma_tchan
  92
  93 struct udma_rflow {
  94         int id;
  95         struct k3_ring *fd_ring; /* Free Descriptor ring */
  96         struct k3_ring *r_ring; /* Receive ring */
  97 };
  98
  99 struct udma_rchan {
 100         void __iomem *reg_rt;
 101
 102         int id;
 103 };
 104
 105 struct udma_oes_offsets {
 106         /* K3 UDMA Output Event Offset */
 107         u32 udma_rchan;
 108
 109         /* BCDMA Output Event Offsets */
 110         u32 bcdma_bchan_data;
 111         u32 bcdma_bchan_ring;
 112         u32 bcdma_tchan_data;
 113         u32 bcdma_tchan_ring;
 114         u32 bcdma_rchan_data;
 115         u32 bcdma_rchan_ring;
 116
 117         /* PKTDMA Output Event Offsets */
 118         u32 pktdma_tchan_flow;
 119         u32 pktdma_rchan_flow;
 120 };
 121
 122 #define UDMA_FLAG_PDMA_ACC32            BIT(0)
 123 #define UDMA_FLAG_PDMA_BURST            BIT(1)
 124 #define UDMA_FLAG_TDTYPE                BIT(2)
 125 #define UDMA_FLAG_BURST_SIZE            BIT(3)
 126 #define UDMA_FLAGS_J7_CLASS             (UDMA_FLAG_PDMA_ACC32 | \
 127                                          UDMA_FLAG_PDMA_BURST | \
 128                                          UDMA_FLAG_TDTYPE | \
 129                                          UDMA_FLAG_BURST_SIZE)
 130
 131 struct udma_match_data {
 132         enum k3_dma_type type;
 133         u32 psil_base;
 134         bool enable_memcpy_support;
 135         u32 flags;
 136         u32 statictr_z_mask;
 137         u8 burst_size[3];
 138         struct udma_soc_data *soc_data;
 139 };
 140
 141 struct udma_soc_data {
 142         struct udma_oes_offsets oes;
 143         u32 bcdma_trigger_event_offset;
 144 };
 145
 146 struct udma_hwdesc {
 147         size_t cppi5_desc_size;
 148         void *cppi5_desc_vaddr;
 149         dma_addr_t cppi5_desc_paddr;
 150
 151         /* TR descriptor internal pointers */
 152         void *tr_req_base;
 153         struct cppi5_tr_resp_t *tr_resp_base;
 154 };
 155
 156 struct udma_rx_flush {
 157         struct udma_hwdesc hwdescs[2];
 158
 159         size_t buffer_size;
 160         void *buffer_vaddr;
 161         dma_addr_t buffer_paddr;
 162 };
 163
 164 struct udma_tpl {
 165         u8 levels;
 166         u32 start_idx[3];
 167 };
 168
 169 struct udma_dev {
 170         struct dma_device ddev;
 171         struct device *dev;
 172         void __iomem *mmrs[MMR_LAST];
 173         const struct udma_match_data *match_data;
 174         const struct udma_soc_data *soc_data;
 175
 176         struct udma_tpl bchan_tpl;
 177         struct udma_tpl tchan_tpl;
 178         struct udma_tpl rchan_tpl;
 179
 180         size_t desc_align; /* alignment to use for descriptors */
 181
 182         struct udma_tisci_rm tisci_rm;
 183
 184         struct k3_ringacc *ringacc;
 185
 186         struct work_struct purge_work;
 187         struct list_head desc_to_purge;
 188         spinlock_t lock;
 189
 190         struct udma_rx_flush rx_flush;
 191
 192         int bchan_cnt;
 193         int tchan_cnt;
 194         int echan_cnt;
 195         int rchan_cnt;
 196         int rflow_cnt;
 197         int tflow_cnt;
 198         unsigned long *bchan_map;
 199         unsigned long *tchan_map;
 200         unsigned long *rchan_map;
 201         unsigned long *rflow_gp_map;
 202         unsigned long *rflow_gp_map_allocated;
 203         unsigned long *rflow_in_use;
 204         unsigned long *tflow_map;
 205
 206         struct udma_bchan *bchans;
 207         struct udma_tchan *tchans;
 208         struct udma_rchan *rchans;
 209         struct udma_rflow *rflows;
 210
 211         struct udma_chan *channels;
 212         u32 psil_base;
 213         u32 atype;
 214         u32 asel;
 215 };
 216
 217 struct udma_desc {
 218         struct virt_dma_desc vd;
 219
 220         bool terminated;
 221
 222         enum dma_transfer_direction dir;
 223
 224         struct udma_static_tr static_tr;
 225         u32 residue;
 226
 227         unsigned int sglen;
 228         unsigned int desc_idx; /* Only used for cyclic in packet mode */
 229         unsigned int tr_idx;
 230
 231         u32 metadata_size;
 232         void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
 233
 234         unsigned int hwdesc_count;
 235         struct udma_hwdesc hwdesc[];
 236 };
 237
 238 enum udma_chan_state {
 239         UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
 240         UDMA_CHAN_IS_ACTIVE, /* Normal operation */
 241         UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
 242 };
 243
 244 struct udma_tx_drain {
 245         struct delayed_work work;
 246         ktime_t tstamp;
 247         u32 residue;
 248 };
 249
 250 struct udma_chan_config {
 251         bool pkt_mode; /* TR or packet */
 252         bool needs_epib; /* EPIB is needed for the communication or not */
 253         u32 psd_size; /* size of Protocol Specific Data */
 254         u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
 255         u32 hdesc_size; /* Size of a packet descriptor in packet mode */
 256         bool notdpkt; /* Suppress sending TDC packet */
 257         int remote_thread_id;
 258         u32 atype;
 259         u32 asel;
 260         u32 src_thread;
 261         u32 dst_thread;
 262         enum psil_endpoint_type ep_type;
 263         bool enable_acc32;
 264         bool enable_burst;
 265         enum udma_tp_level channel_tpl; /* Channel Throughput Level */
 266
 267         u32 tr_trigger_type;
 268         unsigned long tx_flags;
 269
 270         /* PKDMA mapped channel */
 271         int mapped_channel_id;
 272         /* PKTDMA default tflow or rflow for mapped channel */
 273         int default_flow_id;
 274
 275         enum dma_transfer_direction dir;
 276 };
 277
 278 struct udma_chan {
 279         struct virt_dma_chan vc;
 280         struct dma_slave_config cfg;
 281         struct udma_dev *ud;
 282         struct device *dma_dev;
 283         struct udma_desc *desc;
 284         struct udma_desc *terminated_desc;
 285         struct udma_static_tr static_tr;
 286         char *name;
 287
 288         struct udma_bchan *bchan;
 289         struct udma_tchan *tchan;
 290         struct udma_rchan *rchan;
 291         struct udma_rflow *rflow;
 292
 293         bool psil_paired;
 294
 295         int irq_num_ring;
 296         int irq_num_udma;
 297
 298         bool cyclic;
 299         bool paused;
 300
 301         enum udma_chan_state state;
 302         struct completion teardown_completed;
 303
 304         struct udma_tx_drain tx_drain;
 305
 306         /* Channel configuration parameters */
 307         struct udma_chan_config config;
 308         /* Channel configuration parameters (backup) */
 309         struct udma_chan_config backup_config;
 310
 311         /* dmapool for packet mode descriptors */
 312         bool use_dma_pool;
 313         struct dma_pool *hdesc_pool;
 314
 315         u32 id;
 316 };
 317
 318 static inline struct udma_dev *to_udma_dev(struct dma_device *d)
 319 {
 320         return container_of(d, struct udma_dev, ddev);
 321 }
 322
 323 static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
 324 {
 325         return container_of(c, struct udma_chan, vc.chan);
 326 }
 327
 328 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
 329 {
 330         return container_of(t, struct udma_desc, vd.tx);
 331 }
 332
 333 /* Generic register access functions */
 334 static inline u32 udma_read(void __iomem *base, int reg)
 335 {
 336         return readl(base + reg);
 337 }
 338
 339 static inline void udma_write(void __iomem *base, int reg, u32 val)
 340 {
 341         writel(val, base + reg);
 342 }
 343
 344 static inline void udma_update_bits(void __iomem *base, int reg,
 345                                     u32 mask, u32 val)
 346 {
 347         u32 tmp, orig;
 348
 349         orig = readl(base + reg);
 350         tmp = orig & ~mask;
 351         tmp |= (val & mask);
 352
 353         if (tmp != orig)
 354                 writel(tmp, base + reg);
 355 }
 356
 357 /* TCHANRT */
 358 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)
 359 {
 360         if (!uc->tchan)
 361                 return 0;
 362         return udma_read(uc->tchan->reg_rt, reg);
 363 }
 364
 365 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)
 366 {
 367         if (!uc->tchan)
 368                 return;
 369         udma_write(uc->tchan->reg_rt, reg, val);
 370 }
 371
 372 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,
 373                                             u32 mask, u32 val)
 374 {
 375         if (!uc->tchan)
 376                 return;
 377         udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
 378 }
 379
 380 /* RCHANRT */
 381 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)
 382 {
 383         if (!uc->rchan)
 384                 return 0;
 385         return udma_read(uc->rchan->reg_rt, reg);
 386 }
 387
 388 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)
 389 {
 390         if (!uc->rchan)
 391                 return;
 392         udma_write(uc->rchan->reg_rt, reg, val);
 393 }
 394
 395 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,
 396                                             u32 mask, u32 val)
 397 {
 398         if (!uc->rchan)
 399                 return;
 400         udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
 401 }
 402
 403 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
 404 {
 405         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
 406
 407         dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
 408         return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
 409                                               tisci_rm->tisci_navss_dev_id,
 410                                               src_thread, dst_thread);
 411 }
 412
 413 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
 414                              u32 dst_thread)
 415 {
 416         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
 417
 418         dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
 419         return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
 420                                                 tisci_rm->tisci_navss_dev_id,
 421                                                 src_thread, dst_thread);
 422 }
 423
 424 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
 425 {
 426         struct device *chan_dev = &chan->dev->device;
 427
 428         if (asel == 0) {
 429                 /* No special handling for the channel */
 430                 chan->dev->chan_dma_dev = false;
 431
 432                 chan_dev->dma_coherent = false;
 433                 chan_dev->dma_parms = NULL;
 434         } else if (asel == 14 || asel == 15) {
 435                 chan->dev->chan_dma_dev = true;
 436
 437                 chan_dev->dma_coherent = true;
 438                 dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
 439                 chan_dev->dma_parms = chan_dev->parent->dma_parms;
 440         } else {
 441                 dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
 442
 443                 chan_dev->dma_coherent = false;
 444                 chan_dev->dma_parms = NULL;
 445         }
 446 }
 447
 448 static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id)
 449 {
 450         int i;
 451
 452         for (i = 0; i < tpl_map->levels; i++) {
 453                 if (chan_id >= tpl_map->start_idx[i])
 454                         return i;
 455         }
 456
 457         return 0;
 458 }
 459
 460 static void udma_reset_uchan(struct udma_chan *uc)
 461 {
 462         memset(&uc->config, 0, sizeof(uc->config));
 463         uc->config.remote_thread_id = -1;
 464         uc->config.mapped_channel_id = -1;
 465         uc->config.default_flow_id = -1;
 466         uc->state = UDMA_CHAN_IS_IDLE;
 467 }
 468
 469 static void udma_dump_chan_stdata(struct udma_chan *uc)
 470 {
 471         struct device *dev = uc->ud->dev;
 472         u32 offset;
 473         int i;
 474
 475         if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
 476                 dev_dbg(dev, "TCHAN State data:\n");
 477                 for (i = 0; i < 32; i++) {
 478                         offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
 479                         dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
 480                                 udma_tchanrt_read(uc, offset));
 481                 }
 482         }
 483
 484         if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
 485                 dev_dbg(dev, "RCHAN State data:\n");
 486                 for (i = 0; i < 32; i++) {
 487                         offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
 488                         dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
 489                                 udma_rchanrt_read(uc, offset));
 490                 }
 491         }
 492 }
 493
 494 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
 495                                                     int idx)
 496 {
 497         return d->hwdesc[idx].cppi5_desc_paddr;
 498 }
 499
 500 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
 501 {
 502         return d->hwdesc[idx].cppi5_desc_vaddr;
 503 }
 504
 505 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
 506                                                    dma_addr_t paddr)
 507 {
 508         struct udma_desc *d = uc->terminated_desc;
 509
 510         if (d) {
 511                 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
 512                                                                    d->desc_idx);
 513
 514                 if (desc_paddr != paddr)
 515                         d = NULL;
 516         }
 517
 518         if (!d) {
 519                 d = uc->desc;
 520                 if (d) {
 521                         dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
 522                                                                 d->desc_idx);
 523
 524                         if (desc_paddr != paddr)
 525                                 d = NULL;
 526                 }
 527         }
 528
 529         return d;
 530 }
 531
 532 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
 533 {
 534         if (uc->use_dma_pool) {
 535                 int i;
 536
 537                 for (i = 0; i < d->hwdesc_count; i++) {
 538                         if (!d->hwdesc[i].cppi5_desc_vaddr)
 539                                 continue;
 540
 541                         dma_pool_free(uc->hdesc_pool,
 542                                       d->hwdesc[i].cppi5_desc_vaddr,
 543                                       d->hwdesc[i].cppi5_desc_paddr);
 544
 545                         d->hwdesc[i].cppi5_desc_vaddr = NULL;
 546                 }
 547         } else if (d->hwdesc[0].cppi5_desc_vaddr) {
 548                 dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
 549                                   d->hwdesc[0].cppi5_desc_vaddr,
 550                                   d->hwdesc[0].cppi5_desc_paddr);
 551
 552                 d->hwdesc[0].cppi5_desc_vaddr = NULL;
 553         }
 554 }
 555
 556 static void udma_purge_desc_work(struct work_struct *work)
 557 {
 558         struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
 559         struct virt_dma_desc *vd, *_vd;
 560         unsigned long flags;
 561         LIST_HEAD(head);
 562
 563         spin_lock_irqsave(&ud->lock, flags);
 564         list_splice_tail_init(&ud->desc_to_purge, &head);
 565         spin_unlock_irqrestore(&ud->lock, flags);
 566
 567         list_for_each_entry_safe(vd, _vd, &head, node) {
 568                 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
 569                 struct udma_desc *d = to_udma_desc(&vd->tx);
 570
 571                 udma_free_hwdesc(uc, d);
 572                 list_del(&vd->node);
 573                 kfree(d);
 574         }
 575
 576         /* If more to purge, schedule the work again */
 577         if (!list_empty(&ud->desc_to_purge))
 578                 schedule_work(&ud->purge_work);
 579 }
 580
 581 static void udma_desc_free(struct virt_dma_desc *vd)
 582 {
 583         struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
 584         struct udma_chan *uc = to_udma_chan(vd->tx.chan);
 585         struct udma_desc *d = to_udma_desc(&vd->tx);
 586         unsigned long flags;
 587
 588         if (uc->terminated_desc == d)
 589                 uc->terminated_desc = NULL;
 590
 591         if (uc->use_dma_pool) {
 592                 udma_free_hwdesc(uc, d);
 593                 kfree(d);
 594                 return;
 595         }
 596
 597         spin_lock_irqsave(&ud->lock, flags);
 598         list_add_tail(&vd->node, &ud->desc_to_purge);
 599         spin_unlock_irqrestore(&ud->lock, flags);
 600
 601         schedule_work(&ud->purge_work);
 602 }
 603
 604 static bool udma_is_chan_running(struct udma_chan *uc)
 605 {
 606         u32 trt_ctl = 0;
 607         u32 rrt_ctl = 0;
 608
 609         if (uc->tchan)
 610                 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 611         if (uc->rchan)
 612                 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 613
 614         if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
 615                 return true;
 616
 617         return false;
 618 }
 619
 620 static bool udma_is_chan_paused(struct udma_chan *uc)
 621 {
 622         u32 val, pause_mask;
 623
 624         switch (uc->config.dir) {
 625         case DMA_DEV_TO_MEM:
 626                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
 627                 pause_mask = UDMA_PEER_RT_EN_PAUSE;
 628                 break;
 629         case DMA_MEM_TO_DEV:
 630                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
 631                 pause_mask = UDMA_PEER_RT_EN_PAUSE;
 632                 break;
 633         case DMA_MEM_TO_MEM:
 634                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 635                 pause_mask = UDMA_CHAN_RT_CTL_PAUSE;
 636                 break;
 637         default:
 638                 return false;
 639         }
 640
 641         if (val & pause_mask)
 642                 return true;
 643
 644         return false;
 645 }
 646
 647 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
 648 {
 649         return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
 650 }
 651
 652 static int udma_push_to_ring(struct udma_chan *uc, int idx)
 653 {
 654         struct udma_desc *d = uc->desc;
 655         struct k3_ring *ring = NULL;
 656         dma_addr_t paddr;
 657
 658         switch (uc->config.dir) {
 659         case DMA_DEV_TO_MEM:
 660                 ring = uc->rflow->fd_ring;
 661                 break;
 662         case DMA_MEM_TO_DEV:
 663         case DMA_MEM_TO_MEM:
 664                 ring = uc->tchan->t_ring;
 665                 break;
 666         default:
 667                 return -EINVAL;
 668         }
 669
 670         /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
 671         if (idx == -1) {
 672                 paddr = udma_get_rx_flush_hwdesc_paddr(uc);
 673         } else {
 674                 paddr = udma_curr_cppi5_desc_paddr(d, idx);
 675
 676                 wmb(); /* Ensure that writes are not moved over this point */
 677         }
 678
 679         return k3_ringacc_ring_push(ring, &paddr);
 680 }
 681
 682 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
 683 {
 684         if (uc->config.dir != DMA_DEV_TO_MEM)
 685                 return false;
 686
 687         if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
 688                 return true;
 689
 690         return false;
 691 }
 692
 693 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
 694 {
 695         struct k3_ring *ring = NULL;
 696         int ret;
 697
 698         switch (uc->config.dir) {
 699         case DMA_DEV_TO_MEM:
 700                 ring = uc->rflow->r_ring;
 701                 break;
 702         case DMA_MEM_TO_DEV:
 703         case DMA_MEM_TO_MEM:
 704                 ring = uc->tchan->tc_ring;
 705                 break;
 706         default:
 707                 return -ENOENT;
 708         }
 709
 710         ret = k3_ringacc_ring_pop(ring, addr);
 711         if (ret)
 712                 return ret;
 713
 714         rmb(); /* Ensure that reads are not moved before this point */
 715
 716         /* Teardown completion */
 717         if (cppi5_desc_is_tdcm(*addr))
 718                 return 0;
 719
 720         /* Check for flush descriptor */
 721         if (udma_desc_is_rx_flush(uc, *addr))
 722                 return -ENOENT;
 723
 724         return 0;
 725 }
 726
 727 static void udma_reset_rings(struct udma_chan *uc)
 728 {
 729         struct k3_ring *ring1 = NULL;
 730         struct k3_ring *ring2 = NULL;
 731
 732         switch (uc->config.dir) {
 733         case DMA_DEV_TO_MEM:
 734                 if (uc->rchan) {
 735                         ring1 = uc->rflow->fd_ring;
 736                         ring2 = uc->rflow->r_ring;
 737                 }
 738                 break;
 739         case DMA_MEM_TO_DEV:
 740         case DMA_MEM_TO_MEM:
 741                 if (uc->tchan) {
 742                         ring1 = uc->tchan->t_ring;
 743                         ring2 = uc->tchan->tc_ring;
 744                 }
 745                 break;
 746         default:
 747                 break;
 748         }
 749
 750         if (ring1)
 751                 k3_ringacc_ring_reset_dma(ring1,
 752                                           k3_ringacc_ring_get_occ(ring1));
 753         if (ring2)
 754                 k3_ringacc_ring_reset(ring2);
 755
 756         /* make sure we are not leaking memory by stalled descriptor */
 757         if (uc->terminated_desc) {
 758                 udma_desc_free(&uc->terminated_desc->vd);
 759                 uc->terminated_desc = NULL;
 760         }
 761 }
 762
 763 static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val)
 764 {
 765         if (uc->desc->dir == DMA_DEV_TO_MEM) {
 766                 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 767                 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 768                 if (uc->config.ep_type != PSIL_EP_NATIVE)
 769                         udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 770         } else {
 771                 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 772                 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 773                 if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE)
 774                         udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 775         }
 776 }
 777
 778 static void udma_reset_counters(struct udma_chan *uc)
 779 {
 780         u32 val;
 781
 782         if (uc->tchan) {
 783                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
 784                 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 785
 786                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
 787                 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 788
 789                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
 790                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
 791
 792                 if (!uc->bchan) {
 793                         val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
 794                         udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 795                 }
 796         }
 797
 798         if (uc->rchan) {
 799                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
 800                 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 801
 802                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
 803                 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 804
 805                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
 806                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
 807
 808                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
 809                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 810         }
 811 }
 812
 813 static int udma_reset_chan(struct udma_chan *uc, bool hard)
 814 {
 815         switch (uc->config.dir) {
 816         case DMA_DEV_TO_MEM:
 817                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 818                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 819                 break;
 820         case DMA_MEM_TO_DEV:
 821                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 822                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 823                 break;
 824         case DMA_MEM_TO_MEM:
 825                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 826                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 827                 break;
 828         default:
 829                 return -EINVAL;
 830         }
 831
 832         /* Reset all counters */
 833         udma_reset_counters(uc);
 834
 835         /* Hard reset: re-initialize the channel to reset */
 836         if (hard) {
 837                 struct udma_chan_config ucc_backup;
 838                 int ret;
 839
 840                 memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
 841                 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
 842
 843                 /* restore the channel configuration */
 844                 memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
 845                 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
 846                 if (ret)
 847                         return ret;
 848
 849                 /*
 850                  * Setting forced teardown after forced reset helps recovering
 851                  * the rchan.
 852                  */
 853                 if (uc->config.dir == DMA_DEV_TO_MEM)
 854                         udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 855                                            UDMA_CHAN_RT_CTL_EN |
 856                                            UDMA_CHAN_RT_CTL_TDOWN |
 857                                            UDMA_CHAN_RT_CTL_FTDOWN);
 858         }
 859         uc->state = UDMA_CHAN_IS_IDLE;
 860
 861         return 0;
 862 }
 863
 864 static void udma_start_desc(struct udma_chan *uc)
 865 {
 866         struct udma_chan_config *ucc = &uc->config;
 867
 868         if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode &&
 869             (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
 870                 int i;
 871
 872                 /*
 873                  * UDMA only: Push all descriptors to ring for packet mode
 874                  * cyclic or RX
 875                  * PKTDMA supports pre-linked descriptor and cyclic is not
 876                  * supported
 877                  */
 878                 for (i = 0; i < uc->desc->sglen; i++)
 879                         udma_push_to_ring(uc, i);
 880         } else {
 881                 udma_push_to_ring(uc, 0);
 882         }
 883 }
 884
 885 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
 886 {
 887         /* Only PDMAs have staticTR */
 888         if (uc->config.ep_type == PSIL_EP_NATIVE)
 889                 return false;
 890
 891         /* Check if the staticTR configuration has changed for TX */
 892         if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
 893                 return true;
 894
 895         return false;
 896 }
 897
 898 static int udma_start(struct udma_chan *uc)
 899 {
 900         struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
 901
 902         if (!vd) {
 903                 uc->desc = NULL;
 904                 return -ENOENT;
 905         }
 906
 907         list_del(&vd->node);
 908
 909         uc->desc = to_udma_desc(&vd->tx);
 910
 911         /* Channel is already running and does not need reconfiguration */
 912         if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
 913                 udma_start_desc(uc);
 914                 goto out;
 915         }
 916
 917         /* Make sure that we clear the teardown bit, if it is set */
 918         udma_reset_chan(uc, false);
 919
 920         /* Push descriptors before we start the channel */
 921         udma_start_desc(uc);
 922
 923         switch (uc->desc->dir) {
 924         case DMA_DEV_TO_MEM:
 925                 /* Config remote TR */
 926                 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
 927                         u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
 928                                   PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
 929                         const struct udma_match_data *match_data =
 930                                                         uc->ud->match_data;
 931
 932                         if (uc->config.enable_acc32)
 933                                 val |= PDMA_STATIC_TR_XY_ACC32;
 934                         if (uc->config.enable_burst)
 935                                 val |= PDMA_STATIC_TR_XY_BURST;
 936
 937                         udma_rchanrt_write(uc,
 938                                            UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
 939                                            val);
 940
 941                         udma_rchanrt_write(uc,
 942                                 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,
 943                                 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
 944                                                  match_data->statictr_z_mask));
 945
 946                         /* save the current staticTR configuration */
 947                         memcpy(&uc->static_tr, &uc->desc->static_tr,
 948                                sizeof(uc->static_tr));
 949                 }
 950
 951                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 952                                    UDMA_CHAN_RT_CTL_EN);
 953
 954                 /* Enable remote */
 955                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
 956                                    UDMA_PEER_RT_EN_ENABLE);
 957
 958                 break;
 959         case DMA_MEM_TO_DEV:
 960                 /* Config remote TR */
 961                 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
 962                         u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
 963                                   PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
 964
 965                         if (uc->config.enable_acc32)
 966                                 val |= PDMA_STATIC_TR_XY_ACC32;
 967                         if (uc->config.enable_burst)
 968                                 val |= PDMA_STATIC_TR_XY_BURST;
 969
 970                         udma_tchanrt_write(uc,
 971                                            UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
 972                                            val);
 973
 974                         /* save the current staticTR configuration */
 975                         memcpy(&uc->static_tr, &uc->desc->static_tr,
 976                                sizeof(uc->static_tr));
 977                 }
 978
 979                 /* Enable remote */
 980                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
 981                                    UDMA_PEER_RT_EN_ENABLE);
 982
 983                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 984                                    UDMA_CHAN_RT_CTL_EN);
 985
 986                 break;
 987         case DMA_MEM_TO_MEM:
 988                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 989                                    UDMA_CHAN_RT_CTL_EN);
 990                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 991                                    UDMA_CHAN_RT_CTL_EN);
 992
 993                 break;
 994         default:
 995                 return -EINVAL;
 996         }
 997
 998         uc->state = UDMA_CHAN_IS_ACTIVE;
 999 out:
1000
1001         return 0;
1002 }
1003
1004 static int udma_stop(struct udma_chan *uc)
1005 {
1006         enum udma_chan_state old_state = uc->state;
1007
1008         uc->state = UDMA_CHAN_IS_TERMINATING;
1009         reinit_completion(&uc->teardown_completed);
1010
1011         switch (uc->config.dir) {
1012         case DMA_DEV_TO_MEM:
1013                 if (!uc->cyclic && !uc->desc)
1014                         udma_push_to_ring(uc, -1);
1015
1016                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1017                                    UDMA_PEER_RT_EN_ENABLE |
1018                                    UDMA_PEER_RT_EN_TEARDOWN);
1019                 break;
1020         case DMA_MEM_TO_DEV:
1021                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1022                                    UDMA_PEER_RT_EN_ENABLE |
1023                                    UDMA_PEER_RT_EN_FLUSH);
1024                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1025                                    UDMA_CHAN_RT_CTL_EN |
1026                                    UDMA_CHAN_RT_CTL_TDOWN);
1027                 break;
1028         case DMA_MEM_TO_MEM:
1029                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1030                                    UDMA_CHAN_RT_CTL_EN |
1031                                    UDMA_CHAN_RT_CTL_TDOWN);
1032                 break;
1033         default:
1034                 uc->state = old_state;
1035                 complete_all(&uc->teardown_completed);
1036                 return -EINVAL;
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
1043 {
1044         struct udma_desc *d = uc->desc;
1045         struct cppi5_host_desc_t *h_desc;
1046
1047         h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
1048         cppi5_hdesc_reset_to_original(h_desc);
1049         udma_push_to_ring(uc, d->desc_idx);
1050         d->desc_idx = (d->desc_idx + 1) % d->sglen;
1051 }
1052
1053 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
1054 {
1055         struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
1056
1057         memcpy(d->metadata, h_desc->epib, d->metadata_size);
1058 }
1059
1060 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
1061 {
1062         u32 peer_bcnt, bcnt;
1063
1064         /*
1065          * Only TX towards PDMA is affected.
1066          * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer
1067          * completion calculation, consumer must ensure that there is no stale
1068          * data in DMA fabric in this case.
1069          */
1070         if (uc->config.ep_type == PSIL_EP_NATIVE ||
1071             uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
1072                 return true;
1073
1074         peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
1075         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
1076
1077         /* Transfer is incomplete, store current residue and time stamp */
1078         if (peer_bcnt < bcnt) {
1079                 uc->tx_drain.residue = bcnt - peer_bcnt;
1080                 uc->tx_drain.tstamp = ktime_get();
1081                 return false;
1082         }
1083
1084         return true;
1085 }
1086
1087 static void udma_check_tx_completion(struct work_struct *work)
1088 {
1089         struct udma_chan *uc = container_of(work, typeof(*uc),
1090                                             tx_drain.work.work);
1091         bool desc_done = true;
1092         u32 residue_diff;
1093         ktime_t time_diff;
1094         unsigned long delay;
1095
1096         while (1) {
1097                 if (uc->desc) {
1098                         /* Get previous residue and time stamp */
1099                         residue_diff = uc->tx_drain.residue;
1100                         time_diff = uc->tx_drain.tstamp;
1101                         /*
1102                          * Get current residue and time stamp or see if
1103                          * transfer is complete
1104                          */
1105                         desc_done = udma_is_desc_really_done(uc, uc->desc);
1106                 }
1107
1108                 if (!desc_done) {
1109                         /*
1110                          * Find the time delta and residue delta w.r.t
1111                          * previous poll
1112                          */
1113                         time_diff = ktime_sub(uc->tx_drain.tstamp,
1114                                               time_diff) + 1;
1115                         residue_diff -= uc->tx_drain.residue;
1116                         if (residue_diff) {
1117                                 /*
1118                                  * Try to guess when we should check
1119                                  * next time by calculating rate at
1120                                  * which data is being drained at the
1121                                  * peer device
1122                                  */
1123                                 delay = (time_diff / residue_diff) *
1124                                         uc->tx_drain.residue;
1125                         } else {
1126                                 /* No progress, check again in 1 second  */
1127                                 schedule_delayed_work(&uc->tx_drain.work, HZ);
1128                                 break;
1129                         }
1130
1131                         usleep_range(ktime_to_us(delay),
1132                                      ktime_to_us(delay) + 10);
1133                         continue;
1134                 }
1135
1136                 if (uc->desc) {
1137                         struct udma_desc *d = uc->desc;
1138
1139                         udma_decrement_byte_counters(uc, d->residue);
1140                         udma_start(uc);
1141                         vchan_cookie_complete(&d->vd);
1142                         break;
1143                 }
1144
1145                 break;
1146         }
1147 }
1148
1149 static irqreturn_t udma_ring_irq_handler(int irq, void *data)
1150 {
1151         struct udma_chan *uc = data;
1152         struct udma_desc *d;
1153         dma_addr_t paddr = 0;
1154
1155         if (udma_pop_from_ring(uc, &paddr) || !paddr)
1156                 return IRQ_HANDLED;
1157
1158         spin_lock(&uc->vc.lock);
1159
1160         /* Teardown completion message */
1161         if (cppi5_desc_is_tdcm(paddr)) {
1162                 complete_all(&uc->teardown_completed);
1163
1164                 if (uc->terminated_desc) {
1165                         udma_desc_free(&uc->terminated_desc->vd);
1166                         uc->terminated_desc = NULL;
1167                 }
1168
1169                 if (!uc->desc)
1170                         udma_start(uc);
1171
1172                 goto out;
1173         }
1174
1175         d = udma_udma_desc_from_paddr(uc, paddr);
1176
1177         if (d) {
1178                 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
1179                                                                    d->desc_idx);
1180                 if (desc_paddr != paddr) {
1181                         dev_err(uc->ud->dev, "not matching descriptors!\n");
1182                         goto out;
1183                 }
1184
1185                 if (d == uc->desc) {
1186                         /* active descriptor */
1187                         if (uc->cyclic) {
1188                                 udma_cyclic_packet_elapsed(uc);
1189                                 vchan_cyclic_callback(&d->vd);
1190                         } else {
1191                                 if (udma_is_desc_really_done(uc, d)) {
1192                                         udma_decrement_byte_counters(uc, d->residue);
1193                                         udma_start(uc);
1194                                         vchan_cookie_complete(&d->vd);
1195                                 } else {
1196                                         schedule_delayed_work(&uc->tx_drain.work,
1197                                                               0);
1198                                 }
1199                         }
1200                 } else {
1201                         /*
1202                          * terminated descriptor, mark the descriptor as
1203                          * completed to update the channel's cookie marker
1204                          */
1205                         dma_cookie_complete(&d->vd.tx);
1206                 }
1207         }
1208 out:
1209         spin_unlock(&uc->vc.lock);
1210
1211         return IRQ_HANDLED;
1212 }
1213
1214 static irqreturn_t udma_udma_irq_handler(int irq, void *data)
1215 {
1216         struct udma_chan *uc = data;
1217         struct udma_desc *d;
1218
1219         spin_lock(&uc->vc.lock);
1220         d = uc->desc;
1221         if (d) {
1222                 d->tr_idx = (d->tr_idx + 1) % d->sglen;
1223
1224                 if (uc->cyclic) {
1225                         vchan_cyclic_callback(&d->vd);
1226                 } else {
1227                         /* TODO: figure out the real amount of data */
1228                         udma_decrement_byte_counters(uc, d->residue);
1229                         udma_start(uc);
1230                         vchan_cookie_complete(&d->vd);
1231                 }
1232         }
1233
1234         spin_unlock(&uc->vc.lock);
1235
1236         return IRQ_HANDLED;
1237 }
1238
1239 /**
1240  * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
1241  * @ud: UDMA device
1242  * @from: Start the search from this flow id number
1243  * @cnt: Number of consecutive flow ids to allocate
1244  *
1245  * Allocate range of RX flow ids for future use, those flows can be requested
1246  * only using explicit flow id number. if @from is set to -1 it will try to find
1247  * first free range. if @from is positive value it will force allocation only
1248  * of the specified range of flows.
1249  *
1250  * Returns -ENOMEM if can't find free range.
1251  * -EEXIST if requested range is busy.
1252  * -EINVAL if wrong input values passed.
1253  * Returns flow id on success.
1254  */
1255 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1256 {
1257         int start, tmp_from;
1258         DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
1259
1260         tmp_from = from;
1261         if (tmp_from < 0)
1262                 tmp_from = ud->rchan_cnt;
1263         /* default flows can't be allocated and accessible only by id */
1264         if (tmp_from < ud->rchan_cnt)
1265                 return -EINVAL;
1266
1267         if (tmp_from + cnt > ud->rflow_cnt)
1268                 return -EINVAL;
1269
1270         bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
1271                   ud->rflow_cnt);
1272
1273         start = bitmap_find_next_zero_area(tmp,
1274                                            ud->rflow_cnt,
1275                                            tmp_from, cnt, 0);
1276         if (start >= ud->rflow_cnt)
1277                 return -ENOMEM;
1278
1279         if (from >= 0 && start != from)
1280                 return -EEXIST;
1281
1282         bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
1283         return start;
1284 }
1285
1286 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1287 {
1288         if (from < ud->rchan_cnt)
1289                 return -EINVAL;
1290         if (from + cnt > ud->rflow_cnt)
1291                 return -EINVAL;
1292
1293         bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
1294         return 0;
1295 }
1296
1297 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
1298 {
1299         /*
1300          * Attempt to request rflow by ID can be made for any rflow
1301          * if not in use with assumption that caller knows what's doing.
1302          * TI-SCI FW will perform additional permission check ant way, it's
1303          * safe
1304          */
1305
1306         if (id < 0 || id >= ud->rflow_cnt)
1307                 return ERR_PTR(-ENOENT);
1308
1309         if (test_bit(id, ud->rflow_in_use))
1310                 return ERR_PTR(-ENOENT);
1311
1312         if (ud->rflow_gp_map) {
1313                 /* GP rflow has to be allocated first */
1314                 if (!test_bit(id, ud->rflow_gp_map) &&
1315                     !test_bit(id, ud->rflow_gp_map_allocated))
1316                         return ERR_PTR(-EINVAL);
1317         }
1318
1319         dev_dbg(ud->dev, "get rflow%d\n", id);
1320         set_bit(id, ud->rflow_in_use);
1321         return &ud->rflows[id];
1322 }
1323
1324 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
1325 {
1326         if (!test_bit(rflow->id, ud->rflow_in_use)) {
1327                 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
1328                 return;
1329         }
1330
1331         dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
1332         clear_bit(rflow->id, ud->rflow_in_use);
1333 }
1334
1335 #define UDMA_RESERVE_RESOURCE(res)                                      \
1336 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud,     \
1337                                                enum udma_tp_level tpl,  \
1338                                                int id)                  \
1339 {                                                                       \
1340         if (id >= 0) {                                                  \
1341                 if (test_bit(id, ud->res##_map)) {                      \
1342                         dev_err(ud->dev, "res##%d is in use\n", id);    \
1343                         return ERR_PTR(-ENOENT);                        \
1344                 }                                                       \
1345         } else {                                                        \
1346                 int start;                                              \
1347                                                                         \
1348                 if (tpl >= ud->res##_tpl.levels)                        \
1349                         tpl = ud->res##_tpl.levels - 1;                 \
1350                                                                         \
1351                 start = ud->res##_tpl.start_idx[tpl];                   \
1352                                                                         \
1353                 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt,   \
1354                                         start);                         \
1355                 if (id == ud->res##_cnt) {                              \
1356                         return ERR_PTR(-ENOENT);                        \
1357                 }                                                       \
1358         }                                                               \
1359                                                                         \
1360         set_bit(id, ud->res##_map);                                     \
1361         return &ud->res##s[id];                                         \
1362 }
1363
1364 UDMA_RESERVE_RESOURCE(bchan);
1365 UDMA_RESERVE_RESOURCE(tchan);
1366 UDMA_RESERVE_RESOURCE(rchan);
1367
1368 static int bcdma_get_bchan(struct udma_chan *uc)
1369 {
1370         struct udma_dev *ud = uc->ud;
1371         enum udma_tp_level tpl;
1372         int ret;
1373
1374         if (uc->bchan) {
1375                 dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
1376                         uc->id, uc->bchan->id);
1377                 return 0;
1378         }
1379
1380         /*
1381          * Use normal channels for peripherals, and highest TPL channel for
1382          * mem2mem
1383          */
1384         if (uc->config.tr_trigger_type)
1385                 tpl = 0;
1386         else
1387                 tpl = ud->bchan_tpl.levels - 1;
1388
1389         uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
1390         if (IS_ERR(uc->bchan)) {
1391                 ret = PTR_ERR(uc->bchan);
1392                 uc->bchan = NULL;
1393                 return ret;
1394         }
1395
1396         uc->tchan = uc->bchan;
1397
1398         return 0;
1399 }
1400
1401 static int udma_get_tchan(struct udma_chan *uc)
1402 {
1403         struct udma_dev *ud = uc->ud;
1404         int ret;
1405
1406         if (uc->tchan) {
1407                 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
1408                         uc->id, uc->tchan->id);
1409                 return 0;
1410         }
1411
1412         /*
1413          * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1414          * For PKTDMA mapped channels it is configured to a channel which must
1415          * be used to service the peripheral.
1416          */
1417         uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
1418                                          uc->config.mapped_channel_id);
1419         if (IS_ERR(uc->tchan)) {
1420                 ret = PTR_ERR(uc->tchan);
1421                 uc->tchan = NULL;
1422                 return ret;
1423         }
1424
1425         if (ud->tflow_cnt) {
1426                 int tflow_id;
1427
1428                 /* Only PKTDMA have support for tx flows */
1429                 if (uc->config.default_flow_id >= 0)
1430                         tflow_id = uc->config.default_flow_id;
1431                 else
1432                         tflow_id = uc->tchan->id;
1433
1434                 if (test_bit(tflow_id, ud->tflow_map)) {
1435                         dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
1436                         clear_bit(uc->tchan->id, ud->tchan_map);
1437                         uc->tchan = NULL;
1438                         return -ENOENT;
1439                 }
1440
1441                 uc->tchan->tflow_id = tflow_id;
1442                 set_bit(tflow_id, ud->tflow_map);
1443         } else {
1444                 uc->tchan->tflow_id = -1;
1445         }
1446
1447         return 0;
1448 }
1449
1450 static int udma_get_rchan(struct udma_chan *uc)
1451 {
1452         struct udma_dev *ud = uc->ud;
1453         int ret;
1454
1455         if (uc->rchan) {
1456                 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
1457                         uc->id, uc->rchan->id);
1458                 return 0;
1459         }
1460
1461         /*
1462          * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1463          * For PKTDMA mapped channels it is configured to a channel which must
1464          * be used to service the peripheral.
1465          */
1466         uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
1467                                          uc->config.mapped_channel_id);
1468         if (IS_ERR(uc->rchan)) {
1469                 ret = PTR_ERR(uc->rchan);
1470                 uc->rchan = NULL;
1471                 return ret;
1472         }
1473
1474         return 0;
1475 }
1476
1477 static int udma_get_chan_pair(struct udma_chan *uc)
1478 {
1479         struct udma_dev *ud = uc->ud;
1480         int chan_id, end;
1481
1482         if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
1483                 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
1484                          uc->id, uc->tchan->id);
1485                 return 0;
1486         }
1487
1488         if (uc->tchan) {
1489                 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
1490                         uc->id, uc->tchan->id);
1491                 return -EBUSY;
1492         } else if (uc->rchan) {
1493                 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
1494                         uc->id, uc->rchan->id);
1495                 return -EBUSY;
1496         }
1497
1498         /* Can be optimized, but let's have it like this for now */
1499         end = min(ud->tchan_cnt, ud->rchan_cnt);
1500         /*
1501          * Try to use the highest TPL channel pair for MEM_TO_MEM channels
1502          * Note: in UDMAP the channel TPL is symmetric between tchan and rchan
1503          */
1504         chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1];
1505         for (; chan_id < end; chan_id++) {
1506                 if (!test_bit(chan_id, ud->tchan_map) &&
1507                     !test_bit(chan_id, ud->rchan_map))
1508                         break;
1509         }
1510
1511         if (chan_id == end)
1512                 return -ENOENT;
1513
1514         set_bit(chan_id, ud->tchan_map);
1515         set_bit(chan_id, ud->rchan_map);
1516         uc->tchan = &ud->tchans[chan_id];
1517         uc->rchan = &ud->rchans[chan_id];
1518
1519         /* UDMA does not use tx flows */
1520         uc->tchan->tflow_id = -1;
1521
1522         return 0;
1523 }
1524
1525 static int udma_get_rflow(struct udma_chan *uc, int flow_id)
1526 {
1527         struct udma_dev *ud = uc->ud;
1528         int ret;
1529
1530         if (!uc->rchan) {
1531                 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
1532                 return -EINVAL;
1533         }
1534
1535         if (uc->rflow) {
1536                 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
1537                         uc->id, uc->rflow->id);
1538                 return 0;
1539         }
1540
1541         uc->rflow = __udma_get_rflow(ud, flow_id);
1542         if (IS_ERR(uc->rflow)) {
1543                 ret = PTR_ERR(uc->rflow);
1544                 uc->rflow = NULL;
1545                 return ret;
1546         }
1547
1548         return 0;
1549 }
1550
1551 static void bcdma_put_bchan(struct udma_chan *uc)
1552 {
1553         struct udma_dev *ud = uc->ud;
1554
1555         if (uc->bchan) {
1556                 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1557                         uc->bchan->id);
1558                 clear_bit(uc->bchan->id, ud->bchan_map);
1559                 uc->bchan = NULL;
1560                 uc->tchan = NULL;
1561         }
1562 }
1563
1564 static void udma_put_rchan(struct udma_chan *uc)
1565 {
1566         struct udma_dev *ud = uc->ud;
1567
1568         if (uc->rchan) {
1569                 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
1570                         uc->rchan->id);
1571                 clear_bit(uc->rchan->id, ud->rchan_map);
1572                 uc->rchan = NULL;
1573         }
1574 }
1575
1576 static void udma_put_tchan(struct udma_chan *uc)
1577 {
1578         struct udma_dev *ud = uc->ud;
1579
1580         if (uc->tchan) {
1581                 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
1582                         uc->tchan->id);
1583                 clear_bit(uc->tchan->id, ud->tchan_map);
1584
1585                 if (uc->tchan->tflow_id >= 0)
1586                         clear_bit(uc->tchan->tflow_id, ud->tflow_map);
1587
1588                 uc->tchan = NULL;
1589         }
1590 }
1591
1592 static void udma_put_rflow(struct udma_chan *uc)
1593 {
1594         struct udma_dev *ud = uc->ud;
1595
1596         if (uc->rflow) {
1597                 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
1598                         uc->rflow->id);
1599                 __udma_put_rflow(ud, uc->rflow);
1600                 uc->rflow = NULL;
1601         }
1602 }
1603
1604 static void bcdma_free_bchan_resources(struct udma_chan *uc)
1605 {
1606         if (!uc->bchan)
1607                 return;
1608
1609         k3_ringacc_ring_free(uc->bchan->tc_ring);
1610         k3_ringacc_ring_free(uc->bchan->t_ring);
1611         uc->bchan->tc_ring = NULL;
1612         uc->bchan->t_ring = NULL;
1613         k3_configure_chan_coherency(&uc->vc.chan, 0);
1614
1615         bcdma_put_bchan(uc);
1616 }
1617
1618 static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
1619 {
1620         struct k3_ring_cfg ring_cfg;
1621         struct udma_dev *ud = uc->ud;
1622         int ret;
1623
1624         ret = bcdma_get_bchan(uc);
1625         if (ret)
1626                 return ret;
1627
1628         ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1629                                             &uc->bchan->t_ring,
1630                                             &uc->bchan->tc_ring);
1631         if (ret) {
1632                 ret = -EBUSY;
1633                 goto err_ring;
1634         }
1635
1636         memset(&ring_cfg, 0, sizeof(ring_cfg));
1637         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1638         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1639         ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1640
1641         k3_configure_chan_coherency(&uc->vc.chan, ud->asel);
1642         ring_cfg.asel = ud->asel;
1643         ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1644
1645         ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1646         if (ret)
1647                 goto err_ringcfg;
1648
1649         return 0;
1650
1651 err_ringcfg:
1652         k3_ringacc_ring_free(uc->bchan->tc_ring);
1653         uc->bchan->tc_ring = NULL;
1654         k3_ringacc_ring_free(uc->bchan->t_ring);
1655         uc->bchan->t_ring = NULL;
1656         k3_configure_chan_coherency(&uc->vc.chan, 0);
1657 err_ring:
1658         bcdma_put_bchan(uc);
1659
1660         return ret;
1661 }
1662
1663 static void udma_free_tx_resources(struct udma_chan *uc)
1664 {
1665         if (!uc->tchan)
1666                 return;
1667
1668         k3_ringacc_ring_free(uc->tchan->t_ring);
1669         k3_ringacc_ring_free(uc->tchan->tc_ring);
1670         uc->tchan->t_ring = NULL;
1671         uc->tchan->tc_ring = NULL;
1672
1673         udma_put_tchan(uc);
1674 }
1675
1676 static int udma_alloc_tx_resources(struct udma_chan *uc)
1677 {
1678         struct k3_ring_cfg ring_cfg;
1679         struct udma_dev *ud = uc->ud;
1680         struct udma_tchan *tchan;
1681         int ring_idx, ret;
1682
1683         ret = udma_get_tchan(uc);
1684         if (ret)
1685                 return ret;
1686
1687         tchan = uc->tchan;
1688         if (tchan->tflow_id >= 0)
1689                 ring_idx = tchan->tflow_id;
1690         else
1691                 ring_idx = ud->bchan_cnt + tchan->id;
1692
1693         ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
1694                                             &tchan->t_ring,
1695                                             &tchan->tc_ring);
1696         if (ret) {
1697                 ret = -EBUSY;
1698                 goto err_ring;
1699         }
1700
1701         memset(&ring_cfg, 0, sizeof(ring_cfg));
1702         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1703         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1704         if (ud->match_data->type == DMA_TYPE_UDMA) {
1705                 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1706         } else {
1707                 ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1708
1709                 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1710                 ring_cfg.asel = uc->config.asel;
1711                 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1712         }
1713
1714         ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg);
1715         ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg);
1716
1717         if (ret)
1718                 goto err_ringcfg;
1719
1720         return 0;
1721
1722 err_ringcfg:
1723         k3_ringacc_ring_free(uc->tchan->tc_ring);
1724         uc->tchan->tc_ring = NULL;
1725         k3_ringacc_ring_free(uc->tchan->t_ring);
1726         uc->tchan->t_ring = NULL;
1727 err_ring:
1728         udma_put_tchan(uc);
1729
1730         return ret;
1731 }
1732
1733 static void udma_free_rx_resources(struct udma_chan *uc)
1734 {
1735         if (!uc->rchan)
1736                 return;
1737
1738         if (uc->rflow) {
1739                 struct udma_rflow *rflow = uc->rflow;
1740
1741                 k3_ringacc_ring_free(rflow->fd_ring);
1742                 k3_ringacc_ring_free(rflow->r_ring);
1743                 rflow->fd_ring = NULL;
1744                 rflow->r_ring = NULL;
1745
1746                 udma_put_rflow(uc);
1747         }
1748
1749         udma_put_rchan(uc);
1750 }
1751
1752 static int udma_alloc_rx_resources(struct udma_chan *uc)
1753 {
1754         struct udma_dev *ud = uc->ud;
1755         struct k3_ring_cfg ring_cfg;
1756         struct udma_rflow *rflow;
1757         int fd_ring_id;
1758         int ret;
1759
1760         ret = udma_get_rchan(uc);
1761         if (ret)
1762                 return ret;
1763
1764         /* For MEM_TO_MEM we don't need rflow or rings */
1765         if (uc->config.dir == DMA_MEM_TO_MEM)
1766                 return 0;
1767
1768         if (uc->config.default_flow_id >= 0)
1769                 ret = udma_get_rflow(uc, uc->config.default_flow_id);
1770         else
1771                 ret = udma_get_rflow(uc, uc->rchan->id);
1772
1773         if (ret) {
1774                 ret = -EBUSY;
1775                 goto err_rflow;
1776         }
1777
1778         rflow = uc->rflow;
1779         if (ud->tflow_cnt)
1780                 fd_ring_id = ud->tflow_cnt + rflow->id;
1781         else
1782                 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
1783                              uc->rchan->id;
1784
1785         ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
1786                                             &rflow->fd_ring, &rflow->r_ring);
1787         if (ret) {
1788                 ret = -EBUSY;
1789                 goto err_ring;
1790         }
1791
1792         memset(&ring_cfg, 0, sizeof(ring_cfg));
1793
1794         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1795         if (ud->match_data->type == DMA_TYPE_UDMA) {
1796                 if (uc->config.pkt_mode)
1797                         ring_cfg.size = SG_MAX_SEGMENTS;
1798                 else
1799                         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1800
1801                 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1802         } else {
1803                 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1804                 ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1805
1806                 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1807                 ring_cfg.asel = uc->config.asel;
1808                 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1809         }
1810
1811         ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
1812
1813         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1814         ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
1815
1816         if (ret)
1817                 goto err_ringcfg;
1818
1819         return 0;
1820
1821 err_ringcfg:
1822         k3_ringacc_ring_free(rflow->r_ring);
1823         rflow->r_ring = NULL;
1824         k3_ringacc_ring_free(rflow->fd_ring);
1825         rflow->fd_ring = NULL;
1826 err_ring:
1827         udma_put_rflow(uc);
1828 err_rflow:
1829         udma_put_rchan(uc);
1830
1831         return ret;
1832 }
1833
1834 #define TISCI_BCDMA_BCHAN_VALID_PARAMS (                        \
1835         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1836         TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1837
1838 #define TISCI_BCDMA_TCHAN_VALID_PARAMS (                        \
1839         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1840         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1841
1842 #define TISCI_BCDMA_RCHAN_VALID_PARAMS (                        \
1843         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1844
1845 #define TISCI_UDMA_TCHAN_VALID_PARAMS (                         \
1846         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1847         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |      \
1848         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |    \
1849         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |          \
1850         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |      \
1851         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |         \
1852         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |            \
1853         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1854
1855 #define TISCI_UDMA_RCHAN_VALID_PARAMS (                         \
1856         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1857         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |         \
1858         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |            \
1859         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |          \
1860         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |    \
1861         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |     \
1862         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |    \
1863         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |      \
1864         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1865
1866 static int udma_tisci_m2m_channel_config(struct udma_chan *uc)
1867 {
1868         struct udma_dev *ud = uc->ud;
1869         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1870         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1871         struct udma_tchan *tchan = uc->tchan;
1872         struct udma_rchan *rchan = uc->rchan;
1873         u8 burst_size = 0;
1874         int ret;
1875         u8 tpl;
1876
1877         /* Non synchronized - mem to mem type of transfer */
1878         int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1879         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1880         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1881
1882         if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1883                 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id);
1884
1885                 burst_size = ud->match_data->burst_size[tpl];
1886         }
1887
1888         req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1889         req_tx.nav_id = tisci_rm->tisci_dev_id;
1890         req_tx.index = tchan->id;
1891         req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1892         req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1893         req_tx.txcq_qnum = tc_ring;
1894         req_tx.tx_atype = ud->atype;
1895         if (burst_size) {
1896                 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1897                 req_tx.tx_burst_size = burst_size;
1898         }
1899
1900         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1901         if (ret) {
1902                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1903                 return ret;
1904         }
1905
1906         req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
1907         req_rx.nav_id = tisci_rm->tisci_dev_id;
1908         req_rx.index = rchan->id;
1909         req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1910         req_rx.rxcq_qnum = tc_ring;
1911         req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1912         req_rx.rx_atype = ud->atype;
1913         if (burst_size) {
1914                 req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1915                 req_rx.rx_burst_size = burst_size;
1916         }
1917
1918         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1919         if (ret)
1920                 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
1921
1922         return ret;
1923 }
1924
1925 static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1926 {
1927         struct udma_dev *ud = uc->ud;
1928         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1929         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1930         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1931         struct udma_bchan *bchan = uc->bchan;
1932         u8 burst_size = 0;
1933         int ret;
1934         u8 tpl;
1935
1936         if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1937                 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id);
1938
1939                 burst_size = ud->match_data->burst_size[tpl];
1940         }
1941
1942         req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1943         req_tx.nav_id = tisci_rm->tisci_dev_id;
1944         req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1945         req_tx.index = bchan->id;
1946         if (burst_size) {
1947                 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1948                 req_tx.tx_burst_size = burst_size;
1949         }
1950
1951         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1952         if (ret)
1953                 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1954
1955         return ret;
1956 }
1957
1958 static int udma_tisci_tx_channel_config(struct udma_chan *uc)
1959 {
1960         struct udma_dev *ud = uc->ud;
1961         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1962         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1963         struct udma_tchan *tchan = uc->tchan;
1964         int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1965         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1966         u32 mode, fetch_size;
1967         int ret;
1968
1969         if (uc->config.pkt_mode) {
1970                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1971                 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1972                                                    uc->config.psd_size, 0);
1973         } else {
1974                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
1975                 fetch_size = sizeof(struct cppi5_desc_hdr_t);
1976         }
1977
1978         req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1979         req_tx.nav_id = tisci_rm->tisci_dev_id;
1980         req_tx.index = tchan->id;
1981         req_tx.tx_chan_type = mode;
1982         req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1983         req_tx.tx_fetch_size = fetch_size >> 2;
1984         req_tx.txcq_qnum = tc_ring;
1985         req_tx.tx_atype = uc->config.atype;
1986         if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1987             ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1988                 /* wait for peer to complete the teardown for PDMAs */
1989                 req_tx.valid_params |=
1990                                 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1991                 req_tx.tx_tdtype = 1;
1992         }
1993
1994         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1995         if (ret)
1996                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1997
1998         return ret;
1999 }
2000
2001 static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
2002 {
2003         struct udma_dev *ud = uc->ud;
2004         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2005         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2006         struct udma_tchan *tchan = uc->tchan;
2007         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
2008         int ret;
2009
2010         req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
2011         req_tx.nav_id = tisci_rm->tisci_dev_id;
2012         req_tx.index = tchan->id;
2013         req_tx.tx_supr_tdpkt = uc->config.notdpkt;
2014         if (ud->match_data->flags & UDMA_FLAG_TDTYPE) {
2015                 /* wait for peer to complete the teardown for PDMAs */
2016                 req_tx.valid_params |=
2017                                 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
2018                 req_tx.tx_tdtype = 1;
2019         }
2020
2021         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
2022         if (ret)
2023                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
2024
2025         return ret;
2026 }
2027
2028 #define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
2029
2030 static int udma_tisci_rx_channel_config(struct udma_chan *uc)
2031 {
2032         struct udma_dev *ud = uc->ud;
2033         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2034         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2035         struct udma_rchan *rchan = uc->rchan;
2036         int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
2037         int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2038         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2039         struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2040         u32 mode, fetch_size;
2041         int ret;
2042
2043         if (uc->config.pkt_mode) {
2044                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
2045                 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
2046                                                    uc->config.psd_size, 0);
2047         } else {
2048                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
2049                 fetch_size = sizeof(struct cppi5_desc_hdr_t);
2050         }
2051
2052         req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
2053         req_rx.nav_id = tisci_rm->tisci_dev_id;
2054         req_rx.index = rchan->id;
2055         req_rx.rx_fetch_size =  fetch_size >> 2;
2056         req_rx.rxcq_qnum = rx_ring;
2057         req_rx.rx_chan_type = mode;
2058         req_rx.rx_atype = uc->config.atype;
2059
2060         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2061         if (ret) {
2062                 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2063                 return ret;
2064         }
2065
2066         flow_req.valid_params =
2067                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2068                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2069                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
2070                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
2071                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
2072                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
2073                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
2074                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
2075                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
2076                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
2077                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
2078                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
2079                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
2080
2081         flow_req.nav_id = tisci_rm->tisci_dev_id;
2082         flow_req.flow_index = rchan->id;
2083
2084         if (uc->config.needs_epib)
2085                 flow_req.rx_einfo_present = 1;
2086         else
2087                 flow_req.rx_einfo_present = 0;
2088         if (uc->config.psd_size)
2089                 flow_req.rx_psinfo_present = 1;
2090         else
2091                 flow_req.rx_psinfo_present = 0;
2092         flow_req.rx_error_handling = 1;
2093         flow_req.rx_dest_qnum = rx_ring;
2094         flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;
2095         flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;
2096         flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;
2097         flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;
2098         flow_req.rx_fdq0_sz0_qnum = fd_ring;
2099         flow_req.rx_fdq1_qnum = fd_ring;
2100         flow_req.rx_fdq2_qnum = fd_ring;
2101         flow_req.rx_fdq3_qnum = fd_ring;
2102
2103         ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2104
2105         if (ret)
2106                 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
2107
2108         return 0;
2109 }
2110
2111 static int bcdma_tisci_rx_channel_config(struct udma_chan *uc)
2112 {
2113         struct udma_dev *ud = uc->ud;
2114         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2115         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2116         struct udma_rchan *rchan = uc->rchan;
2117         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2118         int ret;
2119
2120         req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2121         req_rx.nav_id = tisci_rm->tisci_dev_id;
2122         req_rx.index = rchan->id;
2123
2124         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2125         if (ret)
2126                 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2127
2128         return ret;
2129 }
2130
2131 static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
2132 {
2133         struct udma_dev *ud = uc->ud;
2134         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2135         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2136         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2137         struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2138         int ret;
2139
2140         req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2141         req_rx.nav_id = tisci_rm->tisci_dev_id;
2142         req_rx.index = uc->rchan->id;
2143
2144         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2145         if (ret) {
2146                 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2147                 return ret;
2148         }
2149
2150         flow_req.valid_params =
2151                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2152                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2153                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2154
2155         flow_req.nav_id = tisci_rm->tisci_dev_id;
2156         flow_req.flow_index = uc->rflow->id;
2157
2158         if (uc->config.needs_epib)
2159                 flow_req.rx_einfo_present = 1;
2160         else
2161                 flow_req.rx_einfo_present = 0;
2162         if (uc->config.psd_size)
2163                 flow_req.rx_psinfo_present = 1;
2164         else
2165                 flow_req.rx_psinfo_present = 0;
2166         flow_req.rx_error_handling = 1;
2167
2168         ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2169
2170         if (ret)
2171                 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2172                         ret);
2173
2174         return ret;
2175 }
2176
2177 static int udma_alloc_chan_resources(struct dma_chan *chan)
2178 {
2179         struct udma_chan *uc = to_udma_chan(chan);
2180         struct udma_dev *ud = to_udma_dev(chan->device);
2181         const struct udma_soc_data *soc_data = ud->soc_data;
2182         struct k3_ring *irq_ring;
2183         u32 irq_udma_idx;
2184         int ret;
2185
2186         uc->dma_dev = ud->dev;
2187
2188         if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
2189                 uc->use_dma_pool = true;
2190                 /* in case of MEM_TO_MEM we have maximum of two TRs */
2191                 if (uc->config.dir == DMA_MEM_TO_MEM) {
2192                         uc->config.hdesc_size = cppi5_trdesc_calc_size(
2193                                         sizeof(struct cppi5_tr_type15_t), 2);
2194                         uc->config.pkt_mode = false;
2195                 }
2196         }
2197
2198         if (uc->use_dma_pool) {
2199                 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2200                                                  uc->config.hdesc_size,
2201                                                  ud->desc_align,
2202                                                  0);
2203                 if (!uc->hdesc_pool) {
2204                         dev_err(ud->ddev.dev,
2205                                 "Descriptor pool allocation failed\n");
2206                         uc->use_dma_pool = false;
2207                         ret = -ENOMEM;
2208                         goto err_cleanup;
2209                 }
2210         }
2211
2212         /*
2213          * Make sure that the completion is in a known state:
2214          * No teardown, the channel is idle
2215          */
2216         reinit_completion(&uc->teardown_completed);
2217         complete_all(&uc->teardown_completed);
2218         uc->state = UDMA_CHAN_IS_IDLE;
2219
2220         switch (uc->config.dir) {
2221         case DMA_MEM_TO_MEM:
2222                 /* Non synchronized - mem to mem type of transfer */
2223                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2224                         uc->id);
2225
2226                 ret = udma_get_chan_pair(uc);
2227                 if (ret)
2228                         goto err_cleanup;
2229
2230                 ret = udma_alloc_tx_resources(uc);
2231                 if (ret) {
2232                         udma_put_rchan(uc);
2233                         goto err_cleanup;
2234                 }
2235
2236                 ret = udma_alloc_rx_resources(uc);
2237                 if (ret) {
2238                         udma_free_tx_resources(uc);
2239                         goto err_cleanup;
2240                 }
2241
2242                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2243                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2244                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2245
2246                 irq_ring = uc->tchan->tc_ring;
2247                 irq_udma_idx = uc->tchan->id;
2248
2249                 ret = udma_tisci_m2m_channel_config(uc);
2250                 break;
2251         case DMA_MEM_TO_DEV:
2252                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2253                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2254                         uc->id);
2255
2256                 ret = udma_alloc_tx_resources(uc);
2257                 if (ret)
2258                         goto err_cleanup;
2259
2260                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2261                 uc->config.dst_thread = uc->config.remote_thread_id;
2262                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2263
2264                 irq_ring = uc->tchan->tc_ring;
2265                 irq_udma_idx = uc->tchan->id;
2266
2267                 ret = udma_tisci_tx_channel_config(uc);
2268                 break;
2269         case DMA_DEV_TO_MEM:
2270                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2271                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2272                         uc->id);
2273
2274                 ret = udma_alloc_rx_resources(uc);
2275                 if (ret)
2276                         goto err_cleanup;
2277
2278                 uc->config.src_thread = uc->config.remote_thread_id;
2279                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2280                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2281
2282                 irq_ring = uc->rflow->r_ring;
2283                 irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id;
2284
2285                 ret = udma_tisci_rx_channel_config(uc);
2286                 break;
2287         default:
2288                 /* Can not happen */
2289                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2290                         __func__, uc->id, uc->config.dir);
2291                 ret = -EINVAL;
2292                 goto err_cleanup;
2293
2294         }
2295
2296         /* check if the channel configuration was successful */
2297         if (ret)
2298                 goto err_res_free;
2299
2300         if (udma_is_chan_running(uc)) {
2301                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2302                 udma_reset_chan(uc, false);
2303                 if (udma_is_chan_running(uc)) {
2304                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2305                         ret = -EBUSY;
2306                         goto err_res_free;
2307                 }
2308         }
2309
2310         /* PSI-L pairing */
2311         ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2312         if (ret) {
2313                 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2314                         uc->config.src_thread, uc->config.dst_thread);
2315                 goto err_res_free;
2316         }
2317
2318         uc->psil_paired = true;
2319
2320         uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
2321         if (uc->irq_num_ring <= 0) {
2322                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2323                         k3_ringacc_get_ring_id(irq_ring));
2324                 ret = -EINVAL;
2325                 goto err_psi_free;
2326         }
2327
2328         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2329                           IRQF_TRIGGER_HIGH, uc->name, uc);
2330         if (ret) {
2331                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2332                 goto err_irq_free;
2333         }
2334
2335         /* Event from UDMA (TR events) only needed for slave TR mode channels */
2336         if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
2337                 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2338                 if (uc->irq_num_udma <= 0) {
2339                         dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
2340                                 irq_udma_idx);
2341                         free_irq(uc->irq_num_ring, uc);
2342                         ret = -EINVAL;
2343                         goto err_irq_free;
2344                 }
2345
2346                 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2347                                   uc->name, uc);
2348                 if (ret) {
2349                         dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
2350                                 uc->id);
2351                         free_irq(uc->irq_num_ring, uc);
2352                         goto err_irq_free;
2353                 }
2354         } else {
2355                 uc->irq_num_udma = 0;
2356         }
2357
2358         udma_reset_rings(uc);
2359
2360         return 0;
2361
2362 err_irq_free:
2363         uc->irq_num_ring = 0;
2364         uc->irq_num_udma = 0;
2365 err_psi_free:
2366         navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2367         uc->psil_paired = false;
2368 err_res_free:
2369         udma_free_tx_resources(uc);
2370         udma_free_rx_resources(uc);
2371 err_cleanup:
2372         udma_reset_uchan(uc);
2373
2374         if (uc->use_dma_pool) {
2375                 dma_pool_destroy(uc->hdesc_pool);
2376                 uc->use_dma_pool = false;
2377         }
2378
2379         return ret;
2380 }
2381
2382 static int bcdma_alloc_chan_resources(struct dma_chan *chan)
2383 {
2384         struct udma_chan *uc = to_udma_chan(chan);
2385         struct udma_dev *ud = to_udma_dev(chan->device);
2386         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2387         u32 irq_udma_idx, irq_ring_idx;
2388         int ret;
2389
2390         /* Only TR mode is supported */
2391         uc->config.pkt_mode = false;
2392
2393         /*
2394          * Make sure that the completion is in a known state:
2395          * No teardown, the channel is idle
2396          */
2397         reinit_completion(&uc->teardown_completed);
2398         complete_all(&uc->teardown_completed);
2399         uc->state = UDMA_CHAN_IS_IDLE;
2400
2401         switch (uc->config.dir) {
2402         case DMA_MEM_TO_MEM:
2403                 /* Non synchronized - mem to mem type of transfer */
2404                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2405                         uc->id);
2406
2407                 ret = bcdma_alloc_bchan_resources(uc);
2408                 if (ret)
2409                         return ret;
2410
2411                 irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring;
2412                 irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data;
2413
2414                 ret = bcdma_tisci_m2m_channel_config(uc);
2415                 break;
2416         case DMA_MEM_TO_DEV:
2417                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2418                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2419                         uc->id);
2420
2421                 ret = udma_alloc_tx_resources(uc);
2422                 if (ret) {
2423                         uc->config.remote_thread_id = -1;
2424                         return ret;
2425                 }
2426
2427                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2428                 uc->config.dst_thread = uc->config.remote_thread_id;
2429                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2430
2431                 irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring;
2432                 irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data;
2433
2434                 ret = bcdma_tisci_tx_channel_config(uc);
2435                 break;
2436         case DMA_DEV_TO_MEM:
2437                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2438                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2439                         uc->id);
2440
2441                 ret = udma_alloc_rx_resources(uc);
2442                 if (ret) {
2443                         uc->config.remote_thread_id = -1;
2444                         return ret;
2445                 }
2446
2447                 uc->config.src_thread = uc->config.remote_thread_id;
2448                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2449                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2450
2451                 irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring;
2452                 irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data;
2453
2454                 ret = bcdma_tisci_rx_channel_config(uc);
2455                 break;
2456         default:
2457                 /* Can not happen */
2458                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2459                         __func__, uc->id, uc->config.dir);
2460                 return -EINVAL;
2461         }
2462
2463         /* check if the channel configuration was successful */
2464         if (ret)
2465                 goto err_res_free;
2466
2467         if (udma_is_chan_running(uc)) {
2468                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2469                 udma_reset_chan(uc, false);
2470                 if (udma_is_chan_running(uc)) {
2471                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2472                         ret = -EBUSY;
2473                         goto err_res_free;
2474                 }
2475         }
2476
2477         uc->dma_dev = dmaengine_get_dma_device(chan);
2478         if (uc->config.dir == DMA_MEM_TO_MEM  && !uc->config.tr_trigger_type) {
2479                 uc->config.hdesc_size = cppi5_trdesc_calc_size(
2480                                         sizeof(struct cppi5_tr_type15_t), 2);
2481
2482                 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2483                                                  uc->config.hdesc_size,
2484                                                  ud->desc_align,
2485                                                  0);
2486                 if (!uc->hdesc_pool) {
2487                         dev_err(ud->ddev.dev,
2488                                 "Descriptor pool allocation failed\n");
2489                         uc->use_dma_pool = false;
2490                         ret = -ENOMEM;
2491                         goto err_res_free;
2492                 }
2493
2494                 uc->use_dma_pool = true;
2495         } else if (uc->config.dir != DMA_MEM_TO_MEM) {
2496                 /* PSI-L pairing */
2497                 ret = navss_psil_pair(ud, uc->config.src_thread,
2498                                       uc->config.dst_thread);
2499                 if (ret) {
2500                         dev_err(ud->dev,
2501                                 "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2502                                 uc->config.src_thread, uc->config.dst_thread);
2503                         goto err_res_free;
2504                 }
2505
2506                 uc->psil_paired = true;
2507         }
2508
2509         uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2510         if (uc->irq_num_ring <= 0) {
2511                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2512                         irq_ring_idx);
2513                 ret = -EINVAL;
2514                 goto err_psi_free;
2515         }
2516
2517         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2518                           IRQF_TRIGGER_HIGH, uc->name, uc);
2519         if (ret) {
2520                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2521                 goto err_irq_free;
2522         }
2523
2524         /* Event from BCDMA (TR events) only needed for slave channels */
2525         if (is_slave_direction(uc->config.dir)) {
2526                 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2527                 if (uc->irq_num_udma <= 0) {
2528                         dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n",
2529                                 irq_udma_idx);
2530                         free_irq(uc->irq_num_ring, uc);
2531                         ret = -EINVAL;
2532                         goto err_irq_free;
2533                 }
2534
2535                 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2536                                   uc->name, uc);
2537                 if (ret) {
2538                         dev_err(ud->dev, "chan%d: BCDMA irq request failed\n",
2539                                 uc->id);
2540                         free_irq(uc->irq_num_ring, uc);
2541                         goto err_irq_free;
2542                 }
2543         } else {
2544                 uc->irq_num_udma = 0;
2545         }
2546
2547         udma_reset_rings(uc);
2548
2549         INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2550                                   udma_check_tx_completion);
2551         return 0;
2552
2553 err_irq_free:
2554         uc->irq_num_ring = 0;
2555         uc->irq_num_udma = 0;
2556 err_psi_free:
2557         if (uc->psil_paired)
2558                 navss_psil_unpair(ud, uc->config.src_thread,
2559                                   uc->config.dst_thread);
2560         uc->psil_paired = false;
2561 err_res_free:
2562         bcdma_free_bchan_resources(uc);
2563         udma_free_tx_resources(uc);
2564         udma_free_rx_resources(uc);
2565
2566         udma_reset_uchan(uc);
2567
2568         if (uc->use_dma_pool) {
2569                 dma_pool_destroy(uc->hdesc_pool);
2570                 uc->use_dma_pool = false;
2571         }
2572
2573         return ret;
2574 }
2575
2576 static int bcdma_router_config(struct dma_chan *chan)
2577 {
2578         struct k3_event_route_data *router_data = chan->route_data;
2579         struct udma_chan *uc = to_udma_chan(chan);
2580         u32 trigger_event;
2581
2582         if (!uc->bchan)
2583                 return -EINVAL;
2584
2585         if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2)
2586                 return -EINVAL;
2587
2588         trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset;
2589         trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1;
2590
2591         return router_data->set_event(router_data->priv, trigger_event);
2592 }
2593
2594 static int pktdma_alloc_chan_resources(struct dma_chan *chan)
2595 {
2596         struct udma_chan *uc = to_udma_chan(chan);
2597         struct udma_dev *ud = to_udma_dev(chan->device);
2598         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2599         u32 irq_ring_idx;
2600         int ret;
2601
2602         /*
2603          * Make sure that the completion is in a known state:
2604          * No teardown, the channel is idle
2605          */
2606         reinit_completion(&uc->teardown_completed);
2607         complete_all(&uc->teardown_completed);
2608         uc->state = UDMA_CHAN_IS_IDLE;
2609
2610         switch (uc->config.dir) {
2611         case DMA_MEM_TO_DEV:
2612                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2613                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2614                         uc->id);
2615
2616                 ret = udma_alloc_tx_resources(uc);
2617                 if (ret) {
2618                         uc->config.remote_thread_id = -1;
2619                         return ret;
2620                 }
2621
2622                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2623                 uc->config.dst_thread = uc->config.remote_thread_id;
2624                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2625
2626                 irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow;
2627
2628                 ret = pktdma_tisci_tx_channel_config(uc);
2629                 break;
2630         case DMA_DEV_TO_MEM:
2631                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2632                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2633                         uc->id);
2634
2635                 ret = udma_alloc_rx_resources(uc);
2636                 if (ret) {
2637                         uc->config.remote_thread_id = -1;
2638                         return ret;
2639                 }
2640
2641                 uc->config.src_thread = uc->config.remote_thread_id;
2642                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2643                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2644
2645                 irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow;
2646
2647                 ret = pktdma_tisci_rx_channel_config(uc);
2648                 break;
2649         default:
2650                 /* Can not happen */
2651                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2652                         __func__, uc->id, uc->config.dir);
2653                 return -EINVAL;
2654         }
2655
2656         /* check if the channel configuration was successful */
2657         if (ret)
2658                 goto err_res_free;
2659
2660         if (udma_is_chan_running(uc)) {
2661                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2662                 udma_reset_chan(uc, false);
2663                 if (udma_is_chan_running(uc)) {
2664                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2665                         ret = -EBUSY;
2666                         goto err_res_free;
2667                 }
2668         }
2669
2670         uc->dma_dev = dmaengine_get_dma_device(chan);
2671         uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev,
2672                                          uc->config.hdesc_size, ud->desc_align,
2673                                          0);
2674         if (!uc->hdesc_pool) {
2675                 dev_err(ud->ddev.dev,
2676                         "Descriptor pool allocation failed\n");
2677                 uc->use_dma_pool = false;
2678                 ret = -ENOMEM;
2679                 goto err_res_free;
2680         }
2681
2682         uc->use_dma_pool = true;
2683
2684         /* PSI-L pairing */
2685         ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2686         if (ret) {
2687                 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2688                         uc->config.src_thread, uc->config.dst_thread);
2689                 goto err_res_free;
2690         }
2691
2692         uc->psil_paired = true;
2693
2694         uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2695         if (uc->irq_num_ring <= 0) {
2696                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2697                         irq_ring_idx);
2698                 ret = -EINVAL;
2699                 goto err_psi_free;
2700         }
2701
2702         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2703                           IRQF_TRIGGER_HIGH, uc->name, uc);
2704         if (ret) {
2705                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2706                 goto err_irq_free;
2707         }
2708
2709         uc->irq_num_udma = 0;
2710
2711         udma_reset_rings(uc);
2712
2713         INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2714                                   udma_check_tx_completion);
2715
2716         if (uc->tchan)
2717                 dev_dbg(ud->dev,
2718                         "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2719                         uc->id, uc->tchan->id, uc->tchan->tflow_id,
2720                         uc->config.remote_thread_id);
2721         else if (uc->rchan)
2722                 dev_dbg(ud->dev,
2723                         "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2724                         uc->id, uc->rchan->id, uc->rflow->id,
2725                         uc->config.remote_thread_id);
2726         return 0;
2727
2728 err_irq_free:
2729         uc->irq_num_ring = 0;
2730 err_psi_free:
2731         navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2732         uc->psil_paired = false;
2733 err_res_free:
2734         udma_free_tx_resources(uc);
2735         udma_free_rx_resources(uc);
2736
2737         udma_reset_uchan(uc);
2738
2739         dma_pool_destroy(uc->hdesc_pool);
2740         uc->use_dma_pool = false;
2741
2742         return ret;
2743 }
2744
2745 static int udma_slave_config(struct dma_chan *chan,
2746                              struct dma_slave_config *cfg)
2747 {
2748         struct udma_chan *uc = to_udma_chan(chan);
2749
2750         memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
2751
2752         return 0;
2753 }
2754
2755 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
2756                                             size_t tr_size, int tr_count,
2757                                             enum dma_transfer_direction dir)
2758 {
2759         struct udma_hwdesc *hwdesc;
2760         struct cppi5_desc_hdr_t *tr_desc;
2761         struct udma_desc *d;
2762         u32 reload_count = 0;
2763         u32 ring_id;
2764
2765         switch (tr_size) {
2766         case 16:
2767         case 32:
2768         case 64:
2769         case 128:
2770                 break;
2771         default:
2772                 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
2773                 return NULL;
2774         }
2775
2776         /* We have only one descriptor containing multiple TRs */
2777         d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
2778         if (!d)
2779                 return NULL;
2780
2781         d->sglen = tr_count;
2782
2783         d->hwdesc_count = 1;
2784         hwdesc = &d->hwdesc[0];
2785
2786         /* Allocate memory for DMA ring descriptor */
2787         if (uc->use_dma_pool) {
2788                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2789                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2790                                                 GFP_NOWAIT,
2791                                                 &hwdesc->cppi5_desc_paddr);
2792         } else {
2793                 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
2794                                                                  tr_count);
2795                 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
2796                                                 uc->ud->desc_align);
2797                 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
2798                                                 hwdesc->cppi5_desc_size,
2799                                                 &hwdesc->cppi5_desc_paddr,
2800                                                 GFP_NOWAIT);
2801         }
2802
2803         if (!hwdesc->cppi5_desc_vaddr) {
2804                 kfree(d);
2805                 return NULL;
2806         }
2807
2808         /* Start of the TR req records */
2809         hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
2810         /* Start address of the TR response array */
2811         hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
2812
2813         tr_desc = hwdesc->cppi5_desc_vaddr;
2814
2815         if (uc->cyclic)
2816                 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
2817
2818         if (dir == DMA_DEV_TO_MEM)
2819                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2820         else
2821                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2822
2823         cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
2824         cppi5_desc_set_pktids(tr_desc, uc->id,
2825                               CPPI5_INFO1_DESC_FLOWID_DEFAULT);
2826         cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
2827
2828         return d;
2829 }
2830
2831 /**
2832  * udma_get_tr_counters - calculate TR counters for a given length
2833  * @len: Length of the trasnfer
2834  * @align_to: Preferred alignment
2835  * @tr0_cnt0: First TR icnt0
2836  * @tr0_cnt1: First TR icnt1
2837  * @tr1_cnt0: Second (if used) TR icnt0
2838  *
2839  * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
2840  * For len >= SZ_64K two TRs are used in a simple way:
2841  * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
2842  * Second TR: the remaining length (tr1_cnt0)
2843  *
2844  * Returns the number of TRs the length needs (1 or 2)
2845  * -EINVAL if the length can not be supported
2846  */
2847 static int udma_get_tr_counters(size_t len, unsigned long align_to,
2848                                 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
2849 {
2850         if (len < SZ_64K) {
2851                 *tr0_cnt0 = len;
2852                 *tr0_cnt1 = 1;
2853
2854                 return 1;
2855         }
2856
2857         if (align_to > 3)
2858                 align_to = 3;
2859
2860 realign:
2861         *tr0_cnt0 = SZ_64K - BIT(align_to);
2862         if (len / *tr0_cnt0 >= SZ_64K) {
2863                 if (align_to) {
2864                         align_to--;
2865                         goto realign;
2866                 }
2867                 return -EINVAL;
2868         }
2869
2870         *tr0_cnt1 = len / *tr0_cnt0;
2871         *tr1_cnt0 = len % *tr0_cnt0;
2872
2873         return 2;
2874 }
2875
2876 static struct udma_desc *
2877 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
2878                       unsigned int sglen, enum dma_transfer_direction dir,
2879                       unsigned long tx_flags, void *context)
2880 {
2881         struct scatterlist *sgent;
2882         struct udma_desc *d;
2883         struct cppi5_tr_type1_t *tr_req = NULL;
2884         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2885         unsigned int i;
2886         size_t tr_size;
2887         int num_tr = 0;
2888         int tr_idx = 0;
2889         u64 asel;
2890
2891         /* estimate the number of TRs we will need */
2892         for_each_sg(sgl, sgent, sglen, i) {
2893                 if (sg_dma_len(sgent) < SZ_64K)
2894                         num_tr++;
2895                 else
2896                         num_tr += 2;
2897         }
2898
2899         /* Now allocate and setup the descriptor. */
2900         tr_size = sizeof(struct cppi5_tr_type1_t);
2901         d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
2902         if (!d)
2903                 return NULL;
2904
2905         d->sglen = sglen;
2906
2907         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
2908                 asel = 0;
2909         else
2910                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
2911
2912         tr_req = d->hwdesc[0].tr_req_base;
2913         for_each_sg(sgl, sgent, sglen, i) {
2914                 dma_addr_t sg_addr = sg_dma_address(sgent);
2915
2916                 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
2917                                               &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
2918                 if (num_tr < 0) {
2919                         dev_err(uc->ud->dev, "size %u is not supported\n",
2920                                 sg_dma_len(sgent));
2921                         udma_free_hwdesc(uc, d);
2922                         kfree(d);
2923                         return NULL;
2924                 }
2925
2926                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
2927                               false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2928                 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
2929
2930                 sg_addr |= asel;
2931                 tr_req[tr_idx].addr = sg_addr;
2932                 tr_req[tr_idx].icnt0 = tr0_cnt0;
2933                 tr_req[tr_idx].icnt1 = tr0_cnt1;
2934                 tr_req[tr_idx].dim1 = tr0_cnt0;
2935                 tr_idx++;
2936
2937                 if (num_tr == 2) {
2938                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
2939                                       false, false,
2940                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2941                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2942                                          CPPI5_TR_CSF_SUPR_EVT);
2943
2944                         tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
2945                         tr_req[tr_idx].icnt0 = tr1_cnt0;
2946                         tr_req[tr_idx].icnt1 = 1;
2947                         tr_req[tr_idx].dim1 = tr1_cnt0;
2948                         tr_idx++;
2949                 }
2950
2951                 d->residue += sg_dma_len(sgent);
2952         }
2953
2954         cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
2955                          CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
2956
2957         return d;
2958 }
2959
2960 static struct udma_desc *
2961 udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,
2962                                 unsigned int sglen,
2963                                 enum dma_transfer_direction dir,
2964                                 unsigned long tx_flags, void *context)
2965 {
2966         struct scatterlist *sgent;
2967         struct cppi5_tr_type15_t *tr_req = NULL;
2968         enum dma_slave_buswidth dev_width;
2969         u32 csf = CPPI5_TR_CSF_SUPR_EVT;
2970         u16 tr_cnt0, tr_cnt1;
2971         dma_addr_t dev_addr;
2972         struct udma_desc *d;
2973         unsigned int i;
2974         size_t tr_size, sg_len;
2975         int num_tr = 0;
2976         int tr_idx = 0;
2977         u32 burst, trigger_size, port_window;
2978         u64 asel;
2979
2980         if (dir == DMA_DEV_TO_MEM) {
2981                 dev_addr = uc->cfg.src_addr;
2982                 dev_width = uc->cfg.src_addr_width;
2983                 burst = uc->cfg.src_maxburst;
2984                 port_window = uc->cfg.src_port_window_size;
2985         } else if (dir == DMA_MEM_TO_DEV) {
2986                 dev_addr = uc->cfg.dst_addr;
2987                 dev_width = uc->cfg.dst_addr_width;
2988                 burst = uc->cfg.dst_maxburst;
2989                 port_window = uc->cfg.dst_port_window_size;
2990         } else {
2991                 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
2992                 return NULL;
2993         }
2994
2995         if (!burst)
2996                 burst = 1;
2997
2998         if (port_window) {
2999                 if (port_window != burst) {
3000                         dev_err(uc->ud->dev,
3001                                 "The burst must be equal to port_window\n");
3002                         return NULL;
3003                 }
3004
3005                 tr_cnt0 = dev_width * port_window;
3006                 tr_cnt1 = 1;
3007         } else {
3008                 tr_cnt0 = dev_width;
3009                 tr_cnt1 = burst;
3010         }
3011         trigger_size = tr_cnt0 * tr_cnt1;
3012
3013         /* estimate the number of TRs we will need */
3014         for_each_sg(sgl, sgent, sglen, i) {
3015                 sg_len = sg_dma_len(sgent);
3016
3017                 if (sg_len % trigger_size) {
3018                         dev_err(uc->ud->dev,
3019                                 "Not aligned SG entry (%zu for %u)\n", sg_len,
3020                                 trigger_size);
3021                         return NULL;
3022                 }
3023
3024                 if (sg_len / trigger_size < SZ_64K)
3025                         num_tr++;
3026                 else
3027                         num_tr += 2;
3028         }
3029
3030         /* Now allocate and setup the descriptor. */
3031         tr_size = sizeof(struct cppi5_tr_type15_t);
3032         d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
3033         if (!d)
3034                 return NULL;
3035
3036         d->sglen = sglen;
3037
3038         if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
3039                 asel = 0;
3040                 csf |= CPPI5_TR_CSF_EOL_ICNT0;
3041         } else {
3042                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3043                 dev_addr |= asel;
3044         }
3045
3046         tr_req = d->hwdesc[0].tr_req_base;
3047         for_each_sg(sgl, sgent, sglen, i) {
3048                 u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;
3049                 dma_addr_t sg_addr = sg_dma_address(sgent);
3050
3051                 sg_len = sg_dma_len(sgent);
3052                 num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,
3053                                               &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);
3054                 if (num_tr < 0) {
3055                         dev_err(uc->ud->dev, "size %zu is not supported\n",
3056                                 sg_len);
3057                         udma_free_hwdesc(uc, d);
3058                         kfree(d);
3059                         return NULL;
3060                 }
3061
3062                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,
3063                               true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3064                 cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
3065                 cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3066                                      uc->config.tr_trigger_type,
3067                                      CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);
3068
3069                 sg_addr |= asel;
3070                 if (dir == DMA_DEV_TO_MEM) {
3071                         tr_req[tr_idx].addr = dev_addr;
3072                         tr_req[tr_idx].icnt0 = tr_cnt0;
3073                         tr_req[tr_idx].icnt1 = tr_cnt1;
3074                         tr_req[tr_idx].icnt2 = tr0_cnt2;
3075                         tr_req[tr_idx].icnt3 = tr0_cnt3;
3076                         tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3077
3078                         tr_req[tr_idx].daddr = sg_addr;
3079                         tr_req[tr_idx].dicnt0 = tr_cnt0;
3080                         tr_req[tr_idx].dicnt1 = tr_cnt1;
3081                         tr_req[tr_idx].dicnt2 = tr0_cnt2;
3082                         tr_req[tr_idx].dicnt3 = tr0_cnt3;
3083                         tr_req[tr_idx].ddim1 = tr_cnt0;
3084                         tr_req[tr_idx].ddim2 = trigger_size;
3085                         tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;
3086                 } else {
3087                         tr_req[tr_idx].addr = sg_addr;
3088                         tr_req[tr_idx].icnt0 = tr_cnt0;
3089                         tr_req[tr_idx].icnt1 = tr_cnt1;
3090                         tr_req[tr_idx].icnt2 = tr0_cnt2;
3091                         tr_req[tr_idx].icnt3 = tr0_cnt3;
3092                         tr_req[tr_idx].dim1 = tr_cnt0;
3093                         tr_req[tr_idx].dim2 = trigger_size;
3094                         tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;
3095
3096                         tr_req[tr_idx].daddr = dev_addr;
3097                         tr_req[tr_idx].dicnt0 = tr_cnt0;
3098                         tr_req[tr_idx].dicnt1 = tr_cnt1;
3099                         tr_req[tr_idx].dicnt2 = tr0_cnt2;
3100                         tr_req[tr_idx].dicnt3 = tr0_cnt3;
3101                         tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3102                 }
3103
3104                 tr_idx++;
3105
3106                 if (num_tr == 2) {
3107                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,
3108                                       false, true,
3109                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3110                         cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
3111                         cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3112                                              uc->config.tr_trigger_type,
3113                                              CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
3114                                              0, 0);
3115
3116                         sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;
3117                         if (dir == DMA_DEV_TO_MEM) {
3118                                 tr_req[tr_idx].addr = dev_addr;
3119                                 tr_req[tr_idx].icnt0 = tr_cnt0;
3120                                 tr_req[tr_idx].icnt1 = tr_cnt1;
3121                                 tr_req[tr_idx].icnt2 = tr1_cnt2;
3122                                 tr_req[tr_idx].icnt3 = 1;
3123                                 tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3124
3125                                 tr_req[tr_idx].daddr = sg_addr;
3126                                 tr_req[tr_idx].dicnt0 = tr_cnt0;
3127                                 tr_req[tr_idx].dicnt1 = tr_cnt1;
3128                                 tr_req[tr_idx].dicnt2 = tr1_cnt2;
3129                                 tr_req[tr_idx].dicnt3 = 1;
3130                                 tr_req[tr_idx].ddim1 = tr_cnt0;
3131                                 tr_req[tr_idx].ddim2 = trigger_size;
3132                         } else {
3133                                 tr_req[tr_idx].addr = sg_addr;
3134                                 tr_req[tr_idx].icnt0 = tr_cnt0;
3135                                 tr_req[tr_idx].icnt1 = tr_cnt1;
3136                                 tr_req[tr_idx].icnt2 = tr1_cnt2;
3137                                 tr_req[tr_idx].icnt3 = 1;
3138                                 tr_req[tr_idx].dim1 = tr_cnt0;
3139                                 tr_req[tr_idx].dim2 = trigger_size;
3140
3141                                 tr_req[tr_idx].daddr = dev_addr;
3142                                 tr_req[tr_idx].dicnt0 = tr_cnt0;
3143                                 tr_req[tr_idx].dicnt1 = tr_cnt1;
3144                                 tr_req[tr_idx].dicnt2 = tr1_cnt2;
3145                                 tr_req[tr_idx].dicnt3 = 1;
3146                                 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3147                         }
3148                         tr_idx++;
3149                 }
3150
3151                 d->residue += sg_len;
3152         }
3153
3154         cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP);
3155
3156         return d;
3157 }
3158
3159 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
3160                                    enum dma_slave_buswidth dev_width,
3161                                    u16 elcnt)
3162 {
3163         if (uc->config.ep_type != PSIL_EP_PDMA_XY)
3164                 return 0;
3165
3166         /* Bus width translates to the element size (ES) */
3167         switch (dev_width) {
3168         case DMA_SLAVE_BUSWIDTH_1_BYTE:
3169                 d->static_tr.elsize = 0;
3170                 break;
3171         case DMA_SLAVE_BUSWIDTH_2_BYTES:
3172                 d->static_tr.elsize = 1;
3173                 break;
3174         case DMA_SLAVE_BUSWIDTH_3_BYTES:
3175                 d->static_tr.elsize = 2;
3176                 break;
3177         case DMA_SLAVE_BUSWIDTH_4_BYTES:
3178                 d->static_tr.elsize = 3;
3179                 break;
3180         case DMA_SLAVE_BUSWIDTH_8_BYTES:
3181                 d->static_tr.elsize = 4;
3182                 break;
3183         default: /* not reached */
3184                 return -EINVAL;
3185         }
3186
3187         d->static_tr.elcnt = elcnt;
3188
3189         /*
3190          * PDMA must to close the packet when the channel is in packet mode.
3191          * For TR mode when the channel is not cyclic we also need PDMA to close
3192          * the packet otherwise the transfer will stall because PDMA holds on
3193          * the data it has received from the peripheral.
3194          */
3195         if (uc->config.pkt_mode || !uc->cyclic) {
3196                 unsigned int div = dev_width * elcnt;
3197
3198                 if (uc->cyclic)
3199                         d->static_tr.bstcnt = d->residue / d->sglen / div;
3200                 else
3201                         d->static_tr.bstcnt = d->residue / div;
3202
3203                 if (uc->config.dir == DMA_DEV_TO_MEM &&
3204                     d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
3205                         return -EINVAL;
3206         } else {
3207                 d->static_tr.bstcnt = 0;
3208         }
3209
3210         return 0;
3211 }
3212
3213 static struct udma_desc *
3214 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
3215                        unsigned int sglen, enum dma_transfer_direction dir,
3216                        unsigned long tx_flags, void *context)
3217 {
3218         struct scatterlist *sgent;
3219         struct cppi5_host_desc_t *h_desc = NULL;
3220         struct udma_desc *d;
3221         u32 ring_id;
3222         unsigned int i;
3223         u64 asel;
3224
3225         d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
3226         if (!d)
3227                 return NULL;
3228
3229         d->sglen = sglen;
3230         d->hwdesc_count = sglen;
3231
3232         if (dir == DMA_DEV_TO_MEM)
3233                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3234         else
3235                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3236
3237         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3238                 asel = 0;
3239         else
3240                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3241
3242         for_each_sg(sgl, sgent, sglen, i) {
3243                 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3244                 dma_addr_t sg_addr = sg_dma_address(sgent);
3245                 struct cppi5_host_desc_t *desc;
3246                 size_t sg_len = sg_dma_len(sgent);
3247
3248                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3249                                                 GFP_NOWAIT,
3250                                                 &hwdesc->cppi5_desc_paddr);
3251                 if (!hwdesc->cppi5_desc_vaddr) {
3252                         dev_err(uc->ud->dev,
3253                                 "descriptor%d allocation failed\n", i);
3254
3255                         udma_free_hwdesc(uc, d);
3256                         kfree(d);
3257                         return NULL;
3258                 }
3259
3260                 d->residue += sg_len;
3261                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3262                 desc = hwdesc->cppi5_desc_vaddr;
3263
3264                 if (i == 0) {
3265                         cppi5_hdesc_init(desc, 0, 0);
3266                         /* Flow and Packed ID */
3267                         cppi5_desc_set_pktids(&desc->hdr, uc->id,
3268                                               CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3269                         cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
3270                 } else {
3271                         cppi5_hdesc_reset_hbdesc(desc);
3272                         cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
3273                 }
3274
3275                 /* attach the sg buffer to the descriptor */
3276                 sg_addr |= asel;
3277                 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
3278
3279                 /* Attach link as host buffer descriptor */
3280                 if (h_desc)
3281                         cppi5_hdesc_link_hbdesc(h_desc,
3282                                                 hwdesc->cppi5_desc_paddr | asel);
3283
3284                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
3285                     dir == DMA_MEM_TO_DEV)
3286                         h_desc = desc;
3287         }
3288
3289         if (d->residue >= SZ_4M) {
3290                 dev_err(uc->ud->dev,
3291                         "%s: Transfer size %u is over the supported 4M range\n",
3292                         __func__, d->residue);
3293                 udma_free_hwdesc(uc, d);
3294                 kfree(d);
3295                 return NULL;
3296         }
3297
3298         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3299         cppi5_hdesc_set_pktlen(h_desc, d->residue);
3300
3301         return d;
3302 }
3303
3304 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
3305                                 void *data, size_t len)
3306 {
3307         struct udma_desc *d = to_udma_desc(desc);
3308         struct udma_chan *uc = to_udma_chan(desc->chan);
3309         struct cppi5_host_desc_t *h_desc;
3310         u32 psd_size = len;
3311         u32 flags = 0;
3312
3313         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3314                 return -ENOTSUPP;
3315
3316         if (!data || len > uc->config.metadata_size)
3317                 return -EINVAL;
3318
3319         if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3320                 return -EINVAL;
3321
3322         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3323         if (d->dir == DMA_MEM_TO_DEV)
3324                 memcpy(h_desc->epib, data, len);
3325
3326         if (uc->config.needs_epib)
3327                 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3328
3329         d->metadata = data;
3330         d->metadata_size = len;
3331         if (uc->config.needs_epib)
3332                 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3333
3334         cppi5_hdesc_update_flags(h_desc, flags);
3335         cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3336
3337         return 0;
3338 }
3339
3340 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
3341                                    size_t *payload_len, size_t *max_len)
3342 {
3343         struct udma_desc *d = to_udma_desc(desc);
3344         struct udma_chan *uc = to_udma_chan(desc->chan);
3345         struct cppi5_host_desc_t *h_desc;
3346
3347         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3348                 return ERR_PTR(-ENOTSUPP);
3349
3350         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3351
3352         *max_len = uc->config.metadata_size;
3353
3354         *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
3355                        CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
3356         *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
3357
3358         return h_desc->epib;
3359 }
3360
3361 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
3362                                  size_t payload_len)
3363 {
3364         struct udma_desc *d = to_udma_desc(desc);
3365         struct udma_chan *uc = to_udma_chan(desc->chan);
3366         struct cppi5_host_desc_t *h_desc;
3367         u32 psd_size = payload_len;
3368         u32 flags = 0;
3369
3370         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3371                 return -ENOTSUPP;
3372
3373         if (payload_len > uc->config.metadata_size)
3374                 return -EINVAL;
3375
3376         if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3377                 return -EINVAL;
3378
3379         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3380
3381         if (uc->config.needs_epib) {
3382                 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3383                 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3384         }
3385
3386         cppi5_hdesc_update_flags(h_desc, flags);
3387         cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3388
3389         return 0;
3390 }
3391
3392 static struct dma_descriptor_metadata_ops metadata_ops = {
3393         .attach = udma_attach_metadata,
3394         .get_ptr = udma_get_metadata_ptr,
3395         .set_len = udma_set_metadata_len,
3396 };
3397
3398 static struct dma_async_tx_descriptor *
3399 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
3400                    unsigned int sglen, enum dma_transfer_direction dir,
3401                    unsigned long tx_flags, void *context)
3402 {
3403         struct udma_chan *uc = to_udma_chan(chan);
3404         enum dma_slave_buswidth dev_width;
3405         struct udma_desc *d;
3406         u32 burst;
3407
3408         if (dir != uc->config.dir &&
3409             (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
3410                 dev_err(chan->device->dev,
3411                         "%s: chan%d is for %s, not supporting %s\n",
3412                         __func__, uc->id,
3413                         dmaengine_get_direction_text(uc->config.dir),
3414                         dmaengine_get_direction_text(dir));
3415                 return NULL;
3416         }
3417
3418         if (dir == DMA_DEV_TO_MEM) {
3419                 dev_width = uc->cfg.src_addr_width;
3420                 burst = uc->cfg.src_maxburst;
3421         } else if (dir == DMA_MEM_TO_DEV) {
3422                 dev_width = uc->cfg.dst_addr_width;
3423                 burst = uc->cfg.dst_maxburst;
3424         } else {
3425                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
3426                 return NULL;
3427         }
3428
3429         if (!burst)
3430                 burst = 1;
3431
3432         uc->config.tx_flags = tx_flags;
3433
3434         if (uc->config.pkt_mode)
3435                 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
3436                                            context);
3437         else if (is_slave_direction(uc->config.dir))
3438                 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
3439                                           context);
3440         else
3441                 d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,
3442                                                     tx_flags, context);
3443
3444         if (!d)
3445                 return NULL;
3446
3447         d->dir = dir;
3448         d->desc_idx = 0;
3449         d->tr_idx = 0;
3450
3451         /* static TR for remote PDMA */
3452         if (udma_configure_statictr(uc, d, dev_width, burst)) {
3453                 dev_err(uc->ud->dev,
3454                         "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3455                         __func__, d->static_tr.bstcnt);
3456
3457                 udma_free_hwdesc(uc, d);
3458                 kfree(d);
3459                 return NULL;
3460         }
3461
3462         if (uc->config.metadata_size)
3463                 d->vd.tx.metadata_ops = &metadata_ops;
3464
3465         return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3466 }
3467
3468 static struct udma_desc *
3469 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
3470                         size_t buf_len, size_t period_len,
3471                         enum dma_transfer_direction dir, unsigned long flags)
3472 {
3473         struct udma_desc *d;
3474         size_t tr_size, period_addr;
3475         struct cppi5_tr_type1_t *tr_req;
3476         unsigned int periods = buf_len / period_len;
3477         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3478         unsigned int i;
3479         int num_tr;
3480
3481         num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
3482                                       &tr0_cnt1, &tr1_cnt0);
3483         if (num_tr < 0) {
3484                 dev_err(uc->ud->dev, "size %zu is not supported\n",
3485                         period_len);
3486                 return NULL;
3487         }
3488
3489         /* Now allocate and setup the descriptor. */
3490         tr_size = sizeof(struct cppi5_tr_type1_t);
3491         d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
3492         if (!d)
3493                 return NULL;
3494
3495         tr_req = d->hwdesc[0].tr_req_base;
3496         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3497                 period_addr = buf_addr;
3498         else
3499                 period_addr = buf_addr |
3500                         ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
3501
3502         for (i = 0; i < periods; i++) {
3503                 int tr_idx = i * num_tr;
3504
3505                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
3506                               false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3507
3508                 tr_req[tr_idx].addr = period_addr;
3509                 tr_req[tr_idx].icnt0 = tr0_cnt0;
3510                 tr_req[tr_idx].icnt1 = tr0_cnt1;
3511                 tr_req[tr_idx].dim1 = tr0_cnt0;
3512
3513                 if (num_tr == 2) {
3514                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3515                                          CPPI5_TR_CSF_SUPR_EVT);
3516                         tr_idx++;
3517
3518                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
3519                                       false, false,
3520                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3521
3522                         tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
3523                         tr_req[tr_idx].icnt0 = tr1_cnt0;
3524                         tr_req[tr_idx].icnt1 = 1;
3525                         tr_req[tr_idx].dim1 = tr1_cnt0;
3526                 }
3527
3528                 if (!(flags & DMA_PREP_INTERRUPT))
3529                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3530                                          CPPI5_TR_CSF_SUPR_EVT);
3531
3532                 period_addr += period_len;
3533         }
3534
3535         return d;
3536 }
3537
3538 static struct udma_desc *
3539 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
3540                          size_t buf_len, size_t period_len,
3541                          enum dma_transfer_direction dir, unsigned long flags)
3542 {
3543         struct udma_desc *d;
3544         u32 ring_id;
3545         int i;
3546         int periods = buf_len / period_len;
3547
3548         if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
3549                 return NULL;
3550
3551         if (period_len >= SZ_4M)
3552                 return NULL;
3553
3554         d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
3555         if (!d)
3556                 return NULL;
3557
3558         d->hwdesc_count = periods;
3559
3560         /* TODO: re-check this... */
3561         if (dir == DMA_DEV_TO_MEM)
3562                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3563         else
3564                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3565
3566         if (uc->ud->match_data->type != DMA_TYPE_UDMA)
3567                 buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3568
3569         for (i = 0; i < periods; i++) {
3570                 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3571                 dma_addr_t period_addr = buf_addr + (period_len * i);
3572                 struct cppi5_host_desc_t *h_desc;
3573
3574                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3575                                                 GFP_NOWAIT,
3576                                                 &hwdesc->cppi5_desc_paddr);
3577                 if (!hwdesc->cppi5_desc_vaddr) {
3578                         dev_err(uc->ud->dev,
3579                                 "descriptor%d allocation failed\n", i);
3580
3581                         udma_free_hwdesc(uc, d);
3582                         kfree(d);
3583                         return NULL;
3584                 }
3585
3586                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3587                 h_desc = hwdesc->cppi5_desc_vaddr;
3588
3589                 cppi5_hdesc_init(h_desc, 0, 0);
3590                 cppi5_hdesc_set_pktlen(h_desc, period_len);
3591
3592                 /* Flow and Packed ID */
3593                 cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
3594                                       CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3595                 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
3596
3597                 /* attach each period to a new descriptor */
3598                 cppi5_hdesc_attach_buf(h_desc,
3599                                        period_addr, period_len,
3600                                        period_addr, period_len);
3601         }
3602
3603         return d;
3604 }
3605
3606 static struct dma_async_tx_descriptor *
3607 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
3608                      size_t period_len, enum dma_transfer_direction dir,
3609                      unsigned long flags)
3610 {
3611         struct udma_chan *uc = to_udma_chan(chan);
3612         enum dma_slave_buswidth dev_width;
3613         struct udma_desc *d;
3614         u32 burst;
3615
3616         if (dir != uc->config.dir) {
3617                 dev_err(chan->device->dev,
3618                         "%s: chan%d is for %s, not supporting %s\n",
3619                         __func__, uc->id,
3620                         dmaengine_get_direction_text(uc->config.dir),
3621                         dmaengine_get_direction_text(dir));
3622                 return NULL;
3623         }
3624
3625         uc->cyclic = true;
3626
3627         if (dir == DMA_DEV_TO_MEM) {
3628                 dev_width = uc->cfg.src_addr_width;
3629                 burst = uc->cfg.src_maxburst;
3630         } else if (dir == DMA_MEM_TO_DEV) {
3631                 dev_width = uc->cfg.dst_addr_width;
3632                 burst = uc->cfg.dst_maxburst;
3633         } else {
3634                 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
3635                 return NULL;
3636         }
3637
3638         if (!burst)
3639                 burst = 1;
3640
3641         if (uc->config.pkt_mode)
3642                 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
3643                                              dir, flags);
3644         else
3645                 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
3646                                             dir, flags);
3647
3648         if (!d)
3649                 return NULL;
3650
3651         d->sglen = buf_len / period_len;
3652
3653         d->dir = dir;
3654         d->residue = buf_len;
3655
3656         /* static TR for remote PDMA */
3657         if (udma_configure_statictr(uc, d, dev_width, burst)) {
3658                 dev_err(uc->ud->dev,
3659                         "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3660                         __func__, d->static_tr.bstcnt);
3661
3662                 udma_free_hwdesc(uc, d);
3663                 kfree(d);
3664                 return NULL;
3665         }
3666
3667         if (uc->config.metadata_size)
3668                 d->vd.tx.metadata_ops = &metadata_ops;
3669
3670         return vchan_tx_prep(&uc->vc, &d->vd, flags);
3671 }
3672
3673 static struct dma_async_tx_descriptor *
3674 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
3675                      size_t len, unsigned long tx_flags)
3676 {
3677         struct udma_chan *uc = to_udma_chan(chan);
3678         struct udma_desc *d;
3679         struct cppi5_tr_type15_t *tr_req;
3680         int num_tr;
3681         size_t tr_size = sizeof(struct cppi5_tr_type15_t);
3682         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3683         u32 csf = CPPI5_TR_CSF_SUPR_EVT;
3684
3685         if (uc->config.dir != DMA_MEM_TO_MEM) {
3686                 dev_err(chan->device->dev,
3687                         "%s: chan%d is for %s, not supporting %s\n",
3688                         __func__, uc->id,
3689                         dmaengine_get_direction_text(uc->config.dir),
3690                         dmaengine_get_direction_text(DMA_MEM_TO_MEM));
3691                 return NULL;
3692         }
3693
3694         num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
3695                                       &tr0_cnt1, &tr1_cnt0);
3696         if (num_tr < 0) {
3697                 dev_err(uc->ud->dev, "size %zu is not supported\n",
3698                         len);
3699                 return NULL;
3700         }
3701
3702         d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
3703         if (!d)
3704                 return NULL;
3705
3706         d->dir = DMA_MEM_TO_MEM;
3707         d->desc_idx = 0;
3708         d->tr_idx = 0;
3709         d->residue = len;
3710
3711         if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
3712                 src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3713                 dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3714         } else {
3715                 csf |= CPPI5_TR_CSF_EOL_ICNT0;
3716         }
3717
3718         tr_req = d->hwdesc[0].tr_req_base;
3719
3720         cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
3721                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3722         cppi5_tr_csf_set(&tr_req[0].flags, csf);
3723
3724         tr_req[0].addr = src;
3725         tr_req[0].icnt0 = tr0_cnt0;
3726         tr_req[0].icnt1 = tr0_cnt1;
3727         tr_req[0].icnt2 = 1;
3728         tr_req[0].icnt3 = 1;
3729         tr_req[0].dim1 = tr0_cnt0;
3730
3731         tr_req[0].daddr = dest;
3732         tr_req[0].dicnt0 = tr0_cnt0;
3733         tr_req[0].dicnt1 = tr0_cnt1;
3734         tr_req[0].dicnt2 = 1;
3735         tr_req[0].dicnt3 = 1;
3736         tr_req[0].ddim1 = tr0_cnt0;
3737
3738         if (num_tr == 2) {
3739                 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
3740                               CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3741                 cppi5_tr_csf_set(&tr_req[1].flags, csf);
3742
3743                 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
3744                 tr_req[1].icnt0 = tr1_cnt0;
3745                 tr_req[1].icnt1 = 1;
3746                 tr_req[1].icnt2 = 1;
3747                 tr_req[1].icnt3 = 1;
3748
3749                 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
3750                 tr_req[1].dicnt0 = tr1_cnt0;
3751                 tr_req[1].dicnt1 = 1;
3752                 tr_req[1].dicnt2 = 1;
3753                 tr_req[1].dicnt3 = 1;
3754         }
3755
3756         cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP);
3757
3758         if (uc->config.metadata_size)
3759                 d->vd.tx.metadata_ops = &metadata_ops;
3760
3761         return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3762 }
3763
3764 static void udma_issue_pending(struct dma_chan *chan)
3765 {
3766         struct udma_chan *uc = to_udma_chan(chan);
3767         unsigned long flags;
3768
3769         spin_lock_irqsave(&uc->vc.lock, flags);
3770
3771         /* If we have something pending and no active descriptor, then */
3772         if (vchan_issue_pending(&uc->vc) && !uc->desc) {
3773                 /*
3774                  * start a descriptor if the channel is NOT [marked as
3775                  * terminating _and_ it is still running (teardown has not
3776                  * completed yet)].
3777                  */
3778                 if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
3779                       udma_is_chan_running(uc)))
3780                         udma_start(uc);
3781         }
3782
3783         spin_unlock_irqrestore(&uc->vc.lock, flags);
3784 }
3785
3786 static enum dma_status udma_tx_status(struct dma_chan *chan,
3787                                       dma_cookie_t cookie,
3788                                       struct dma_tx_state *txstate)
3789 {
3790         struct udma_chan *uc = to_udma_chan(chan);
3791         enum dma_status ret;
3792         unsigned long flags;
3793
3794         spin_lock_irqsave(&uc->vc.lock, flags);
3795
3796         ret = dma_cookie_status(chan, cookie, txstate);
3797
3798         if (!udma_is_chan_running(uc))
3799                 ret = DMA_COMPLETE;
3800
3801         if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))
3802                 ret = DMA_PAUSED;
3803
3804         if (ret == DMA_COMPLETE || !txstate)
3805                 goto out;
3806
3807         if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
3808                 u32 peer_bcnt = 0;
3809                 u32 bcnt = 0;
3810                 u32 residue = uc->desc->residue;
3811                 u32 delay = 0;
3812
3813                 if (uc->desc->dir == DMA_MEM_TO_DEV) {
3814                         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
3815
3816                         if (uc->config.ep_type != PSIL_EP_NATIVE) {
3817                                 peer_bcnt = udma_tchanrt_read(uc,
3818                                                 UDMA_CHAN_RT_PEER_BCNT_REG);
3819
3820                                 if (bcnt > peer_bcnt)
3821                                         delay = bcnt - peer_bcnt;
3822                         }
3823                 } else if (uc->desc->dir == DMA_DEV_TO_MEM) {
3824                         bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3825
3826                         if (uc->config.ep_type != PSIL_EP_NATIVE) {
3827                                 peer_bcnt = udma_rchanrt_read(uc,
3828                                                 UDMA_CHAN_RT_PEER_BCNT_REG);
3829
3830                                 if (peer_bcnt > bcnt)
3831                                         delay = peer_bcnt - bcnt;
3832                         }
3833                 } else {
3834                         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3835                 }
3836
3837                 if (bcnt && !(bcnt % uc->desc->residue))
3838                         residue = 0;
3839                 else
3840                         residue -= bcnt % uc->desc->residue;
3841
3842                 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
3843                         ret = DMA_COMPLETE;
3844                         delay = 0;
3845                 }
3846
3847                 dma_set_residue(txstate, residue);
3848                 dma_set_in_flight_bytes(txstate, delay);
3849
3850         } else {
3851                 ret = DMA_COMPLETE;
3852         }
3853
3854 out:
3855         spin_unlock_irqrestore(&uc->vc.lock, flags);
3856         return ret;
3857 }
3858
3859 static int udma_pause(struct dma_chan *chan)
3860 {
3861         struct udma_chan *uc = to_udma_chan(chan);
3862
3863         /* pause the channel */
3864         switch (uc->config.dir) {
3865         case DMA_DEV_TO_MEM:
3866                 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3867                                          UDMA_PEER_RT_EN_PAUSE,
3868                                          UDMA_PEER_RT_EN_PAUSE);
3869                 break;
3870         case DMA_MEM_TO_DEV:
3871                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3872                                          UDMA_PEER_RT_EN_PAUSE,
3873                                          UDMA_PEER_RT_EN_PAUSE);
3874                 break;
3875         case DMA_MEM_TO_MEM:
3876                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3877                                          UDMA_CHAN_RT_CTL_PAUSE,
3878                                          UDMA_CHAN_RT_CTL_PAUSE);
3879                 break;
3880         default:
3881                 return -EINVAL;
3882         }
3883
3884         return 0;
3885 }
3886
3887 static int udma_resume(struct dma_chan *chan)
3888 {
3889         struct udma_chan *uc = to_udma_chan(chan);
3890
3891         /* resume the channel */
3892         switch (uc->config.dir) {
3893         case DMA_DEV_TO_MEM:
3894                 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3895                                          UDMA_PEER_RT_EN_PAUSE, 0);
3896
3897                 break;
3898         case DMA_MEM_TO_DEV:
3899                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3900                                          UDMA_PEER_RT_EN_PAUSE, 0);
3901                 break;
3902         case DMA_MEM_TO_MEM:
3903                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3904                                          UDMA_CHAN_RT_CTL_PAUSE, 0);
3905                 break;
3906         default:
3907                 return -EINVAL;
3908         }
3909
3910         return 0;
3911 }
3912
3913 static int udma_terminate_all(struct dma_chan *chan)
3914 {
3915         struct udma_chan *uc = to_udma_chan(chan);
3916         unsigned long flags;
3917         LIST_HEAD(head);
3918
3919         spin_lock_irqsave(&uc->vc.lock, flags);
3920
3921         if (udma_is_chan_running(uc))
3922                 udma_stop(uc);
3923
3924         if (uc->desc) {
3925                 uc->terminated_desc = uc->desc;
3926                 uc->desc = NULL;
3927                 uc->terminated_desc->terminated = true;
3928                 cancel_delayed_work(&uc->tx_drain.work);
3929         }
3930
3931         uc->paused = false;
3932
3933         vchan_get_all_descriptors(&uc->vc, &head);
3934         spin_unlock_irqrestore(&uc->vc.lock, flags);
3935         vchan_dma_desc_free_list(&uc->vc, &head);
3936
3937         return 0;
3938 }
3939
3940 static void udma_synchronize(struct dma_chan *chan)
3941 {
3942         struct udma_chan *uc = to_udma_chan(chan);
3943         unsigned long timeout = msecs_to_jiffies(1000);
3944
3945         vchan_synchronize(&uc->vc);
3946
3947         if (uc->state == UDMA_CHAN_IS_TERMINATING) {
3948                 timeout = wait_for_completion_timeout(&uc->teardown_completed,
3949                                                       timeout);
3950                 if (!timeout) {
3951                         dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
3952                                  uc->id);
3953                         udma_dump_chan_stdata(uc);
3954                         udma_reset_chan(uc, true);
3955                 }
3956         }
3957
3958         udma_reset_chan(uc, false);
3959         if (udma_is_chan_running(uc))
3960                 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
3961
3962         cancel_delayed_work_sync(&uc->tx_drain.work);
3963         udma_reset_rings(uc);
3964 }
3965
3966 static void udma_desc_pre_callback(struct virt_dma_chan *vc,
3967                                    struct virt_dma_desc *vd,
3968                                    struct dmaengine_result *result)
3969 {
3970         struct udma_chan *uc = to_udma_chan(&vc->chan);
3971         struct udma_desc *d;
3972
3973         if (!vd)
3974                 return;
3975
3976         d = to_udma_desc(&vd->tx);
3977
3978         if (d->metadata_size)
3979                 udma_fetch_epib(uc, d);
3980
3981         /* Provide residue information for the client */
3982         if (result) {
3983                 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
3984
3985                 if (cppi5_desc_get_type(desc_vaddr) ==
3986                     CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
3987                         result->residue = d->residue -
3988                                           cppi5_hdesc_get_pktlen(desc_vaddr);
3989                         if (result->residue)
3990                                 result->result = DMA_TRANS_ABORTED;
3991                         else
3992                                 result->result = DMA_TRANS_NOERROR;
3993                 } else {
3994                         result->residue = 0;
3995                         result->result = DMA_TRANS_NOERROR;
3996                 }
3997         }
3998 }
3999
4000 /*
4001  * This tasklet handles the completion of a DMA descriptor by
4002  * calling its callback and freeing it.
4003  */
4004 static void udma_vchan_complete(struct tasklet_struct *t)
4005 {
4006         struct virt_dma_chan *vc = from_tasklet(vc, t, task);
4007         struct virt_dma_desc *vd, *_vd;
4008         struct dmaengine_desc_callback cb;
4009         LIST_HEAD(head);
4010
4011         spin_lock_irq(&vc->lock);
4012         list_splice_tail_init(&vc->desc_completed, &head);
4013         vd = vc->cyclic;
4014         if (vd) {
4015                 vc->cyclic = NULL;
4016                 dmaengine_desc_get_callback(&vd->tx, &cb);
4017         } else {
4018                 memset(&cb, 0, sizeof(cb));
4019         }
4020         spin_unlock_irq(&vc->lock);
4021
4022         udma_desc_pre_callback(vc, vd, NULL);
4023         dmaengine_desc_callback_invoke(&cb, NULL);
4024
4025         list_for_each_entry_safe(vd, _vd, &head, node) {
4026                 struct dmaengine_result result;
4027
4028                 dmaengine_desc_get_callback(&vd->tx, &cb);
4029
4030                 list_del(&vd->node);
4031
4032                 udma_desc_pre_callback(vc, vd, &result);
4033                 dmaengine_desc_callback_invoke(&cb, &result);
4034
4035                 vchan_vdesc_fini(vd);
4036         }
4037 }
4038
4039 static void udma_free_chan_resources(struct dma_chan *chan)
4040 {
4041         struct udma_chan *uc = to_udma_chan(chan);
4042         struct udma_dev *ud = to_udma_dev(chan->device);
4043
4044         udma_terminate_all(chan);
4045         if (uc->terminated_desc) {
4046                 udma_reset_chan(uc, false);
4047                 udma_reset_rings(uc);
4048         }
4049
4050         cancel_delayed_work_sync(&uc->tx_drain.work);
4051
4052         if (uc->irq_num_ring > 0) {
4053                 free_irq(uc->irq_num_ring, uc);
4054
4055                 uc->irq_num_ring = 0;
4056         }
4057         if (uc->irq_num_udma > 0) {
4058                 free_irq(uc->irq_num_udma, uc);
4059
4060                 uc->irq_num_udma = 0;
4061         }
4062
4063         /* Release PSI-L pairing */
4064         if (uc->psil_paired) {
4065                 navss_psil_unpair(ud, uc->config.src_thread,
4066                                   uc->config.dst_thread);
4067                 uc->psil_paired = false;
4068         }
4069
4070         vchan_free_chan_resources(&uc->vc);
4071         tasklet_kill(&uc->vc.task);
4072
4073         bcdma_free_bchan_resources(uc);
4074         udma_free_tx_resources(uc);
4075         udma_free_rx_resources(uc);
4076         udma_reset_uchan(uc);
4077
4078         if (uc->use_dma_pool) {
4079                 dma_pool_destroy(uc->hdesc_pool);
4080                 uc->use_dma_pool = false;
4081         }
4082 }
4083
4084 static struct platform_driver udma_driver;
4085 static struct platform_driver bcdma_driver;
4086 static struct platform_driver pktdma_driver;
4087
4088 struct udma_filter_param {
4089         int remote_thread_id;
4090         u32 atype;
4091         u32 asel;
4092         u32 tr_trigger_type;
4093 };
4094
4095 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
4096 {
4097         struct udma_chan_config *ucc;
4098         struct psil_endpoint_config *ep_config;
4099         struct udma_filter_param *filter_param;
4100         struct udma_chan *uc;
4101         struct udma_dev *ud;
4102
4103         if (chan->device->dev->driver != &udma_driver.driver &&
4104             chan->device->dev->driver != &bcdma_driver.driver &&
4105             chan->device->dev->driver != &pktdma_driver.driver)
4106                 return false;
4107
4108         uc = to_udma_chan(chan);
4109         ucc = &uc->config;
4110         ud = uc->ud;
4111         filter_param = param;
4112
4113         if (filter_param->atype > 2) {
4114                 dev_err(ud->dev, "Invalid channel atype: %u\n",
4115                         filter_param->atype);
4116                 return false;
4117         }
4118
4119         if (filter_param->asel > 15) {
4120                 dev_err(ud->dev, "Invalid channel asel: %u\n",
4121                         filter_param->asel);
4122                 return false;
4123         }
4124
4125         ucc->remote_thread_id = filter_param->remote_thread_id;
4126         ucc->atype = filter_param->atype;
4127         ucc->asel = filter_param->asel;
4128         ucc->tr_trigger_type = filter_param->tr_trigger_type;
4129
4130         if (ucc->tr_trigger_type) {
4131                 ucc->dir = DMA_MEM_TO_MEM;
4132                 goto triggered_bchan;
4133         } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) {
4134                 ucc->dir = DMA_MEM_TO_DEV;
4135         } else {
4136                 ucc->dir = DMA_DEV_TO_MEM;
4137         }
4138
4139         ep_config = psil_get_ep_config(ucc->remote_thread_id);
4140         if (IS_ERR(ep_config)) {
4141                 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
4142                         ucc->remote_thread_id);
4143                 ucc->dir = DMA_MEM_TO_MEM;
4144                 ucc->remote_thread_id = -1;
4145                 ucc->atype = 0;
4146                 ucc->asel = 0;
4147                 return false;
4148         }
4149
4150         if (ud->match_data->type == DMA_TYPE_BCDMA &&
4151             ep_config->pkt_mode) {
4152                 dev_err(ud->dev,
4153                         "Only TR mode is supported (psi-l thread 0x%04x)\n",
4154                         ucc->remote_thread_id);
4155                 ucc->dir = DMA_MEM_TO_MEM;
4156                 ucc->remote_thread_id = -1;
4157                 ucc->atype = 0;
4158                 ucc->asel = 0;
4159                 return false;
4160         }
4161
4162         ucc->pkt_mode = ep_config->pkt_mode;
4163         ucc->channel_tpl = ep_config->channel_tpl;
4164         ucc->notdpkt = ep_config->notdpkt;
4165         ucc->ep_type = ep_config->ep_type;
4166
4167         if (ud->match_data->type == DMA_TYPE_PKTDMA &&
4168             ep_config->mapped_channel_id >= 0) {
4169                 ucc->mapped_channel_id = ep_config->mapped_channel_id;
4170                 ucc->default_flow_id = ep_config->default_flow_id;
4171         } else {
4172                 ucc->mapped_channel_id = -1;
4173                 ucc->default_flow_id = -1;
4174         }
4175
4176         if (ucc->ep_type != PSIL_EP_NATIVE) {
4177                 const struct udma_match_data *match_data = ud->match_data;
4178
4179                 if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
4180                         ucc->enable_acc32 = ep_config->pdma_acc32;
4181                 if (match_data->flags & UDMA_FLAG_PDMA_BURST)
4182                         ucc->enable_burst = ep_config->pdma_burst;
4183         }
4184
4185         ucc->needs_epib = ep_config->needs_epib;
4186         ucc->psd_size = ep_config->psd_size;
4187         ucc->metadata_size =
4188                         (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
4189                         ucc->psd_size;
4190
4191         if (ucc->pkt_mode)
4192                 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
4193                                  ucc->metadata_size, ud->desc_align);
4194
4195         dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
4196                 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
4197
4198         return true;
4199
4200 triggered_bchan:
4201         dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id,
4202                 ucc->tr_trigger_type);
4203
4204         return true;
4205
4206 }
4207
4208 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
4209                                       struct of_dma *ofdma)
4210 {
4211         struct udma_dev *ud = ofdma->of_dma_data;
4212         dma_cap_mask_t mask = ud->ddev.cap_mask;
4213         struct udma_filter_param filter_param;
4214         struct dma_chan *chan;
4215
4216         if (ud->match_data->type == DMA_TYPE_BCDMA) {
4217                 if (dma_spec->args_count != 3)
4218                         return NULL;
4219
4220                 filter_param.tr_trigger_type = dma_spec->args[0];
4221                 filter_param.remote_thread_id = dma_spec->args[1];
4222                 filter_param.asel = dma_spec->args[2];
4223                 filter_param.atype = 0;
4224         } else {
4225                 if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
4226                         return NULL;
4227
4228                 filter_param.remote_thread_id = dma_spec->args[0];
4229                 filter_param.tr_trigger_type = 0;
4230                 if (dma_spec->args_count == 2) {
4231                         if (ud->match_data->type == DMA_TYPE_UDMA) {
4232                                 filter_param.atype = dma_spec->args[1];
4233                                 filter_param.asel = 0;
4234                         } else {
4235                                 filter_param.atype = 0;
4236                                 filter_param.asel = dma_spec->args[1];
4237                         }
4238                 } else {
4239                         filter_param.atype = 0;
4240                         filter_param.asel = 0;
4241                 }
4242         }
4243
4244         chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,
4245                                      ofdma->of_node);
4246         if (!chan) {
4247                 dev_err(ud->dev, "get channel fail in %s.\n", __func__);
4248                 return ERR_PTR(-EINVAL);
4249         }
4250
4251         return chan;
4252 }
4253
4254 static struct udma_match_data am654_main_data = {
4255         .type = DMA_TYPE_UDMA,
4256         .psil_base = 0x1000,
4257         .enable_memcpy_support = true,
4258         .statictr_z_mask = GENMASK(11, 0),
4259         .burst_size = {
4260                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4261                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4262                 0, /* No UH Channels */
4263         },
4264 };
4265
4266 static struct udma_match_data am654_mcu_data = {
4267         .type = DMA_TYPE_UDMA,
4268         .psil_base = 0x6000,
4269         .enable_memcpy_support = false,
4270         .statictr_z_mask = GENMASK(11, 0),
4271         .burst_size = {
4272                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4273                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4274                 0, /* No UH Channels */
4275         },
4276 };
4277
4278 static struct udma_match_data j721e_main_data = {
4279         .type = DMA_TYPE_UDMA,
4280         .psil_base = 0x1000,
4281         .enable_memcpy_support = true,
4282         .flags = UDMA_FLAGS_J7_CLASS,
4283         .statictr_z_mask = GENMASK(23, 0),
4284         .burst_size = {
4285                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4286                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */
4287                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */
4288         },
4289 };
4290
4291 static struct udma_match_data j721e_mcu_data = {
4292         .type = DMA_TYPE_UDMA,
4293         .psil_base = 0x6000,
4294         .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
4295         .flags = UDMA_FLAGS_J7_CLASS,
4296         .statictr_z_mask = GENMASK(23, 0),
4297         .burst_size = {
4298                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4299                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */
4300                 0, /* No UH Channels */
4301         },
4302 };
4303
4304 static struct udma_soc_data am62a_dmss_csi_soc_data = {
4305         .oes = {
4306                 .bcdma_rchan_data = 0xe00,
4307                 .bcdma_rchan_ring = 0x1000,
4308         },
4309 };
4310
4311 static struct udma_match_data am62a_bcdma_csirx_data = {
4312         .type = DMA_TYPE_BCDMA,
4313         .psil_base = 0x3100,
4314         .enable_memcpy_support = false,
4315         .burst_size = {
4316                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4317                 0, /* No H Channels */
4318                 0, /* No UH Channels */
4319         },
4320         .soc_data = &am62a_dmss_csi_soc_data,
4321 };
4322
4323 static struct udma_match_data am64_bcdma_data = {
4324         .type = DMA_TYPE_BCDMA,
4325         .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
4326         .enable_memcpy_support = true, /* Supported via bchan */
4327         .flags = UDMA_FLAGS_J7_CLASS,
4328         .statictr_z_mask = GENMASK(23, 0),
4329         .burst_size = {
4330                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4331                 0, /* No H Channels */
4332                 0, /* No UH Channels */
4333         },
4334 };
4335
4336 static struct udma_match_data am64_pktdma_data = {
4337         .type = DMA_TYPE_PKTDMA,
4338         .psil_base = 0x1000,
4339         .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */
4340         .flags = UDMA_FLAGS_J7_CLASS,
4341         .statictr_z_mask = GENMASK(23, 0),
4342         .burst_size = {
4343                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4344                 0, /* No H Channels */
4345                 0, /* No UH Channels */
4346         },
4347 };
4348
4349 static const struct of_device_id udma_of_match[] = {
4350         {
4351                 .compatible = "ti,am654-navss-main-udmap",
4352                 .data = &am654_main_data,
4353         },
4354         {
4355                 .compatible = "ti,am654-navss-mcu-udmap",
4356                 .data = &am654_mcu_data,
4357         }, {
4358                 .compatible = "ti,j721e-navss-main-udmap",
4359                 .data = &j721e_main_data,
4360         }, {
4361                 .compatible = "ti,j721e-navss-mcu-udmap",
4362                 .data = &j721e_mcu_data,
4363         },
4364         {
4365                 .compatible = "ti,am64-dmss-bcdma",
4366                 .data = &am64_bcdma_data,
4367         },
4368         {
4369                 .compatible = "ti,am64-dmss-pktdma",
4370                 .data = &am64_pktdma_data,
4371         },
4372         {
4373                 .compatible = "ti,am62a-dmss-bcdma-csirx",
4374                 .data = &am62a_bcdma_csirx_data,
4375         },
4376         { /* Sentinel */ },
4377 };
4378
4379 static struct udma_soc_data am654_soc_data = {
4380         .oes = {
4381                 .udma_rchan = 0x200,
4382         },
4383 };
4384
4385 static struct udma_soc_data j721e_soc_data = {
4386         .oes = {
4387                 .udma_rchan = 0x400,
4388         },
4389 };
4390
4391 static struct udma_soc_data j7200_soc_data = {
4392         .oes = {
4393                 .udma_rchan = 0x80,
4394         },
4395 };
4396
4397 static struct udma_soc_data am64_soc_data = {
4398         .oes = {
4399                 .bcdma_bchan_data = 0x2200,
4400                 .bcdma_bchan_ring = 0x2400,
4401                 .bcdma_tchan_data = 0x2800,
4402                 .bcdma_tchan_ring = 0x2a00,
4403                 .bcdma_rchan_data = 0x2e00,
4404                 .bcdma_rchan_ring = 0x3000,
4405                 .pktdma_tchan_flow = 0x1200,
4406                 .pktdma_rchan_flow = 0x1600,
4407         },
4408         .bcdma_trigger_event_offset = 0xc400,
4409 };
4410
4411 static const struct soc_device_attribute k3_soc_devices[] = {
4412         { .family = "AM65X", .data = &am654_soc_data },
4413         { .family = "J721E", .data = &j721e_soc_data },
4414         { .family = "J7200", .data = &j7200_soc_data },
4415         { .family = "AM64X", .data = &am64_soc_data },
4416         { .family = "J721S2", .data = &j721e_soc_data},
4417         { .family = "AM62X", .data = &am64_soc_data },
4418         { .family = "AM62AX", .data = &am64_soc_data },
4419         { .family = "J784S4", .data = &j721e_soc_data },
4420         { /* sentinel */ }
4421 };
4422
4423 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
4424 {
4425         u32 cap2, cap3, cap4;
4426         int i;
4427
4428         ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]);
4429         if (IS_ERR(ud->mmrs[MMR_GCFG]))
4430                 return PTR_ERR(ud->mmrs[MMR_GCFG]);
4431
4432         cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
4433         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4434
4435         switch (ud->match_data->type) {
4436         case DMA_TYPE_UDMA:
4437                 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4438                 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4439                 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
4440                 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4441                 break;
4442         case DMA_TYPE_BCDMA:
4443                 ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2);
4444                 ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2);
4445                 ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2);
4446                 ud->rflow_cnt = ud->rchan_cnt;
4447                 break;
4448         case DMA_TYPE_PKTDMA:
4449                 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4450                 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4451                 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4452                 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4453                 ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4);
4454                 break;
4455         default:
4456                 return -EINVAL;
4457         }
4458
4459         for (i = 1; i < MMR_LAST; i++) {
4460                 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
4461                         continue;
4462                 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
4463                         continue;
4464                 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
4465                         continue;
4466
4467                 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
4468                 if (IS_ERR(ud->mmrs[i]))
4469                         return PTR_ERR(ud->mmrs[i]);
4470         }
4471
4472         return 0;
4473 }
4474
4475 static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map,
4476                                       struct ti_sci_resource_desc *rm_desc,
4477                                       char *name)
4478 {
4479         bitmap_clear(map, rm_desc->start, rm_desc->num);
4480         bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec);
4481         dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name,
4482                 rm_desc->start, rm_desc->num, rm_desc->start_sec,
4483                 rm_desc->num_sec);
4484 }
4485
4486 static const char * const range_names[] = {
4487         [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
4488         [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
4489         [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
4490         [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
4491         [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
4492 };
4493
4494 static int udma_setup_resources(struct udma_dev *ud)
4495 {
4496         int ret, i, j;
4497         struct device *dev = ud->dev;
4498         struct ti_sci_resource *rm_res, irq_res;
4499         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4500         u32 cap3;
4501
4502         /* Set up the throughput level start indexes */
4503         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4504         if (of_device_is_compatible(dev->of_node,
4505                                     "ti,am654-navss-main-udmap")) {
4506                 ud->tchan_tpl.levels = 2;
4507                 ud->tchan_tpl.start_idx[0] = 8;
4508         } else if (of_device_is_compatible(dev->of_node,
4509                                            "ti,am654-navss-mcu-udmap")) {
4510                 ud->tchan_tpl.levels = 2;
4511                 ud->tchan_tpl.start_idx[0] = 2;
4512         } else if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4513                 ud->tchan_tpl.levels = 3;
4514                 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4515                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4516         } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4517                 ud->tchan_tpl.levels = 2;
4518                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4519         } else {
4520                 ud->tchan_tpl.levels = 1;
4521         }
4522
4523         ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4524         ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4525         ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4526
4527         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4528                                            sizeof(unsigned long), GFP_KERNEL);
4529         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4530                                   GFP_KERNEL);
4531         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4532                                            sizeof(unsigned long), GFP_KERNEL);
4533         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4534                                   GFP_KERNEL);
4535         ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
4536                                               sizeof(unsigned long),
4537                                               GFP_KERNEL);
4538         ud->rflow_gp_map_allocated = devm_kcalloc(dev,
4539                                                   BITS_TO_LONGS(ud->rflow_cnt),
4540                                                   sizeof(unsigned long),
4541                                                   GFP_KERNEL);
4542         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4543                                         sizeof(unsigned long),
4544                                         GFP_KERNEL);
4545         ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4546                                   GFP_KERNEL);
4547
4548         if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
4549             !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
4550             !ud->rflows || !ud->rflow_in_use)
4551                 return -ENOMEM;
4552
4553         /*
4554          * RX flows with the same Ids as RX channels are reserved to be used
4555          * as default flows if remote HW can't generate flow_ids. Those
4556          * RX flows can be requested only explicitly by id.
4557          */
4558         bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
4559
4560         /* by default no GP rflows are assigned to Linux */
4561         bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
4562
4563         /* Get resource ranges from tisci */
4564         for (i = 0; i < RM_RANGE_LAST; i++) {
4565                 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
4566                         continue;
4567
4568                 tisci_rm->rm_ranges[i] =
4569                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4570                                                     tisci_rm->tisci_dev_id,
4571                                                     (char *)range_names[i]);
4572         }
4573
4574         /* tchan ranges */
4575         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4576         if (IS_ERR(rm_res)) {
4577                 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4578                 irq_res.sets = 1;
4579         } else {
4580                 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4581                 for (i = 0; i < rm_res->sets; i++)
4582                         udma_mark_resource_ranges(ud, ud->tchan_map,
4583                                                   &rm_res->desc[i], "tchan");
4584                 irq_res.sets = rm_res->sets;
4585         }
4586
4587         /* rchan and matching default flow ranges */
4588         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4589         if (IS_ERR(rm_res)) {
4590                 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4591                 irq_res.sets++;
4592         } else {
4593                 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4594                 for (i = 0; i < rm_res->sets; i++)
4595                         udma_mark_resource_ranges(ud, ud->rchan_map,
4596                                                   &rm_res->desc[i], "rchan");
4597                 irq_res.sets += rm_res->sets;
4598         }
4599
4600         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4601         if (!irq_res.desc)
4602                 return -ENOMEM;
4603         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4604         if (IS_ERR(rm_res)) {
4605                 irq_res.desc[0].start = 0;
4606                 irq_res.desc[0].num = ud->tchan_cnt;
4607                 i = 1;
4608         } else {
4609                 for (i = 0; i < rm_res->sets; i++) {
4610                         irq_res.desc[i].start = rm_res->desc[i].start;
4611                         irq_res.desc[i].num = rm_res->desc[i].num;
4612                         irq_res.desc[i].start_sec = rm_res->desc[i].start_sec;
4613                         irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
4614                 }
4615         }
4616         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4617         if (IS_ERR(rm_res)) {
4618                 irq_res.desc[i].start = 0;
4619                 irq_res.desc[i].num = ud->rchan_cnt;
4620         } else {
4621                 for (j = 0; j < rm_res->sets; j++, i++) {
4622                         if (rm_res->desc[j].num) {
4623                                 irq_res.desc[i].start = rm_res->desc[j].start +
4624                                                 ud->soc_data->oes.udma_rchan;
4625                                 irq_res.desc[i].num = rm_res->desc[j].num;
4626                         }
4627                         if (rm_res->desc[j].num_sec) {
4628                                 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4629                                                 ud->soc_data->oes.udma_rchan;
4630                                 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4631                         }
4632                 }
4633         }
4634         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4635         kfree(irq_res.desc);
4636         if (ret) {
4637                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4638                 return ret;
4639         }
4640
4641         /* GP rflow ranges */
4642         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4643         if (IS_ERR(rm_res)) {
4644                 /* all gp flows are assigned exclusively to Linux */
4645                 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
4646                              ud->rflow_cnt - ud->rchan_cnt);
4647         } else {
4648                 for (i = 0; i < rm_res->sets; i++)
4649                         udma_mark_resource_ranges(ud, ud->rflow_gp_map,
4650                                                   &rm_res->desc[i], "gp-rflow");
4651         }
4652
4653         return 0;
4654 }
4655
4656 static int bcdma_setup_resources(struct udma_dev *ud)
4657 {
4658         int ret, i, j;
4659         struct device *dev = ud->dev;
4660         struct ti_sci_resource *rm_res, irq_res;
4661         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4662         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4663         u32 cap;
4664
4665         /* Set up the throughput level start indexes */
4666         cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4667         if (BCDMA_CAP3_UBCHAN_CNT(cap)) {
4668                 ud->bchan_tpl.levels = 3;
4669                 ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap);
4670                 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4671         } else if (BCDMA_CAP3_HBCHAN_CNT(cap)) {
4672                 ud->bchan_tpl.levels = 2;
4673                 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4674         } else {
4675                 ud->bchan_tpl.levels = 1;
4676         }
4677
4678         cap = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4679         if (BCDMA_CAP4_URCHAN_CNT(cap)) {
4680                 ud->rchan_tpl.levels = 3;
4681                 ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap);
4682                 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4683         } else if (BCDMA_CAP4_HRCHAN_CNT(cap)) {
4684                 ud->rchan_tpl.levels = 2;
4685                 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4686         } else {
4687                 ud->rchan_tpl.levels = 1;
4688         }
4689
4690         if (BCDMA_CAP4_UTCHAN_CNT(cap)) {
4691                 ud->tchan_tpl.levels = 3;
4692                 ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap);
4693                 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4694         } else if (BCDMA_CAP4_HTCHAN_CNT(cap)) {
4695                 ud->tchan_tpl.levels = 2;
4696                 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4697         } else {
4698                 ud->tchan_tpl.levels = 1;
4699         }
4700
4701         ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
4702                                            sizeof(unsigned long), GFP_KERNEL);
4703         ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
4704                                   GFP_KERNEL);
4705         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4706                                            sizeof(unsigned long), GFP_KERNEL);
4707         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4708                                   GFP_KERNEL);
4709         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4710                                            sizeof(unsigned long), GFP_KERNEL);
4711         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4712                                   GFP_KERNEL);
4713         /* BCDMA do not really have flows, but the driver expect it */
4714         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt),
4715                                         sizeof(unsigned long),
4716                                         GFP_KERNEL);
4717         ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
4718                                   GFP_KERNEL);
4719
4720         if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
4721             !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans ||
4722             !ud->rflows)
4723                 return -ENOMEM;
4724
4725         /* Get resource ranges from tisci */
4726         for (i = 0; i < RM_RANGE_LAST; i++) {
4727                 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
4728                         continue;
4729                 if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0)
4730                         continue;
4731                 if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0)
4732                         continue;
4733                 if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0)
4734                         continue;
4735
4736                 tisci_rm->rm_ranges[i] =
4737                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4738                                                     tisci_rm->tisci_dev_id,
4739                                                     (char *)range_names[i]);
4740         }
4741
4742         irq_res.sets = 0;
4743
4744         /* bchan ranges */
4745         if (ud->bchan_cnt) {
4746                 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4747                 if (IS_ERR(rm_res)) {
4748                         bitmap_zero(ud->bchan_map, ud->bchan_cnt);
4749                         irq_res.sets++;
4750                 } else {
4751                         bitmap_fill(ud->bchan_map, ud->bchan_cnt);
4752                         for (i = 0; i < rm_res->sets; i++)
4753                                 udma_mark_resource_ranges(ud, ud->bchan_map,
4754                                                           &rm_res->desc[i],
4755                                                           "bchan");
4756                         irq_res.sets += rm_res->sets;
4757                 }
4758         }
4759
4760         /* tchan ranges */
4761         if (ud->tchan_cnt) {
4762                 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4763                 if (IS_ERR(rm_res)) {
4764                         bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4765                         irq_res.sets += 2;
4766                 } else {
4767                         bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4768                         for (i = 0; i < rm_res->sets; i++)
4769                                 udma_mark_resource_ranges(ud, ud->tchan_map,
4770                                                           &rm_res->desc[i],
4771                                                           "tchan");
4772                         irq_res.sets += rm_res->sets * 2;
4773                 }
4774         }
4775
4776         /* rchan ranges */
4777         if (ud->rchan_cnt) {
4778                 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4779                 if (IS_ERR(rm_res)) {
4780                         bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4781                         irq_res.sets += 2;
4782                 } else {
4783                         bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4784                         for (i = 0; i < rm_res->sets; i++)
4785                                 udma_mark_resource_ranges(ud, ud->rchan_map,
4786                                                           &rm_res->desc[i],
4787                                                           "rchan");
4788                         irq_res.sets += rm_res->sets * 2;
4789                 }
4790         }
4791
4792         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4793         if (!irq_res.desc)
4794                 return -ENOMEM;
4795         if (ud->bchan_cnt) {
4796                 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4797                 if (IS_ERR(rm_res)) {
4798                         irq_res.desc[0].start = oes->bcdma_bchan_ring;
4799                         irq_res.desc[0].num = ud->bchan_cnt;
4800                         i = 1;
4801                 } else {
4802                         for (i = 0; i < rm_res->sets; i++) {
4803                                 irq_res.desc[i].start = rm_res->desc[i].start +
4804                                                         oes->bcdma_bchan_ring;
4805                                 irq_res.desc[i].num = rm_res->desc[i].num;
4806                         }
4807                 }
4808         } else {
4809                 i = 0;
4810         }
4811
4812         if (ud->tchan_cnt) {
4813                 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4814                 if (IS_ERR(rm_res)) {
4815                         irq_res.desc[i].start = oes->bcdma_tchan_data;
4816                         irq_res.desc[i].num = ud->tchan_cnt;
4817                         irq_res.desc[i + 1].start = oes->bcdma_tchan_ring;
4818                         irq_res.desc[i + 1].num = ud->tchan_cnt;
4819                         i += 2;
4820                 } else {
4821                         for (j = 0; j < rm_res->sets; j++, i += 2) {
4822                                 irq_res.desc[i].start = rm_res->desc[j].start +
4823                                                         oes->bcdma_tchan_data;
4824                                 irq_res.desc[i].num = rm_res->desc[j].num;
4825
4826                                 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4827                                                         oes->bcdma_tchan_ring;
4828                                 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4829                         }
4830                 }
4831         }
4832         if (ud->rchan_cnt) {
4833                 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4834                 if (IS_ERR(rm_res)) {
4835                         irq_res.desc[i].start = oes->bcdma_rchan_data;
4836                         irq_res.desc[i].num = ud->rchan_cnt;
4837                         irq_res.desc[i + 1].start = oes->bcdma_rchan_ring;
4838                         irq_res.desc[i + 1].num = ud->rchan_cnt;
4839                         i += 2;
4840                 } else {
4841                         for (j = 0; j < rm_res->sets; j++, i += 2) {
4842                                 irq_res.desc[i].start = rm_res->desc[j].start +
4843                                                         oes->bcdma_rchan_data;
4844                                 irq_res.desc[i].num = rm_res->desc[j].num;
4845
4846                                 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4847                                                         oes->bcdma_rchan_ring;
4848                                 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4849                         }
4850                 }
4851         }
4852
4853         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4854         kfree(irq_res.desc);
4855         if (ret) {
4856                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4857                 return ret;
4858         }
4859
4860         return 0;
4861 }
4862
4863 static int pktdma_setup_resources(struct udma_dev *ud)
4864 {
4865         int ret, i, j;
4866         struct device *dev = ud->dev;
4867         struct ti_sci_resource *rm_res, irq_res;
4868         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4869         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4870         u32 cap3;
4871
4872         /* Set up the throughput level start indexes */
4873         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4874         if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4875                 ud->tchan_tpl.levels = 3;
4876                 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4877                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4878         } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4879                 ud->tchan_tpl.levels = 2;
4880                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4881         } else {
4882                 ud->tchan_tpl.levels = 1;
4883         }
4884
4885         ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4886         ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4887         ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4888
4889         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4890                                            sizeof(unsigned long), GFP_KERNEL);
4891         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4892                                   GFP_KERNEL);
4893         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4894                                            sizeof(unsigned long), GFP_KERNEL);
4895         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4896                                   GFP_KERNEL);
4897         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4898                                         sizeof(unsigned long),
4899                                         GFP_KERNEL);
4900         ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4901                                   GFP_KERNEL);
4902         ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
4903                                            sizeof(unsigned long), GFP_KERNEL);
4904
4905         if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
4906             !ud->rchans || !ud->rflows || !ud->rflow_in_use)
4907                 return -ENOMEM;
4908
4909         /* Get resource ranges from tisci */
4910         for (i = 0; i < RM_RANGE_LAST; i++) {
4911                 if (i == RM_RANGE_BCHAN)
4912                         continue;
4913
4914                 tisci_rm->rm_ranges[i] =
4915                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4916                                                     tisci_rm->tisci_dev_id,
4917                                                     (char *)range_names[i]);
4918         }
4919
4920         /* tchan ranges */
4921         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4922         if (IS_ERR(rm_res)) {
4923                 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4924         } else {
4925                 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4926                 for (i = 0; i < rm_res->sets; i++)
4927                         udma_mark_resource_ranges(ud, ud->tchan_map,
4928                                                   &rm_res->desc[i], "tchan");
4929         }
4930
4931         /* rchan ranges */
4932         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4933         if (IS_ERR(rm_res)) {
4934                 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4935         } else {
4936                 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4937                 for (i = 0; i < rm_res->sets; i++)
4938                         udma_mark_resource_ranges(ud, ud->rchan_map,
4939                                                   &rm_res->desc[i], "rchan");
4940         }
4941
4942         /* rflow ranges */
4943         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4944         if (IS_ERR(rm_res)) {
4945                 /* all rflows are assigned exclusively to Linux */
4946                 bitmap_zero(ud->rflow_in_use, ud->rflow_cnt);
4947                 irq_res.sets = 1;
4948         } else {
4949                 bitmap_fill(ud->rflow_in_use, ud->rflow_cnt);
4950                 for (i = 0; i < rm_res->sets; i++)
4951                         udma_mark_resource_ranges(ud, ud->rflow_in_use,
4952                                                   &rm_res->desc[i], "rflow");
4953                 irq_res.sets = rm_res->sets;
4954         }
4955
4956         /* tflow ranges */
4957         rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4958         if (IS_ERR(rm_res)) {
4959                 /* all tflows are assigned exclusively to Linux */
4960                 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
4961                 irq_res.sets++;
4962         } else {
4963                 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
4964                 for (i = 0; i < rm_res->sets; i++)
4965                         udma_mark_resource_ranges(ud, ud->tflow_map,
4966                                                   &rm_res->desc[i], "tflow");
4967                 irq_res.sets += rm_res->sets;
4968         }
4969
4970         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4971         if (!irq_res.desc)
4972                 return -ENOMEM;
4973         rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4974         if (IS_ERR(rm_res)) {
4975                 irq_res.desc[0].start = oes->pktdma_tchan_flow;
4976                 irq_res.desc[0].num = ud->tflow_cnt;
4977                 i = 1;
4978         } else {
4979                 for (i = 0; i < rm_res->sets; i++) {
4980                         irq_res.desc[i].start = rm_res->desc[i].start +
4981                                                 oes->pktdma_tchan_flow;
4982                         irq_res.desc[i].num = rm_res->desc[i].num;
4983                 }
4984         }
4985         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4986         if (IS_ERR(rm_res)) {
4987                 irq_res.desc[i].start = oes->pktdma_rchan_flow;
4988                 irq_res.desc[i].num = ud->rflow_cnt;
4989         } else {
4990                 for (j = 0; j < rm_res->sets; j++, i++) {
4991                         irq_res.desc[i].start = rm_res->desc[j].start +
4992                                                 oes->pktdma_rchan_flow;
4993                         irq_res.desc[i].num = rm_res->desc[j].num;
4994                 }
4995         }
4996         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4997         kfree(irq_res.desc);
4998         if (ret) {
4999                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
5000                 return ret;
5001         }
5002
5003         return 0;
5004 }
5005
5006 static int setup_resources(struct udma_dev *ud)
5007 {
5008         struct device *dev = ud->dev;
5009         int ch_count, ret;
5010
5011         switch (ud->match_data->type) {
5012         case DMA_TYPE_UDMA:
5013                 ret = udma_setup_resources(ud);
5014                 break;
5015         case DMA_TYPE_BCDMA:
5016                 ret = bcdma_setup_resources(ud);
5017                 break;
5018         case DMA_TYPE_PKTDMA:
5019                 ret = pktdma_setup_resources(ud);
5020                 break;
5021         default:
5022                 return -EINVAL;
5023         }
5024
5025         if (ret)
5026                 return ret;
5027
5028         ch_count  = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
5029         if (ud->bchan_cnt)
5030                 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
5031         ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
5032         ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
5033         if (!ch_count)
5034                 return -ENODEV;
5035
5036         ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
5037                                     GFP_KERNEL);
5038         if (!ud->channels)
5039                 return -ENOMEM;
5040
5041         switch (ud->match_data->type) {
5042         case DMA_TYPE_UDMA:
5043                 dev_info(dev,
5044                          "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
5045                          ch_count,
5046                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5047                                                        ud->tchan_cnt),
5048                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5049                                                        ud->rchan_cnt),
5050                          ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
5051                                                        ud->rflow_cnt));
5052                 break;
5053         case DMA_TYPE_BCDMA:
5054                 dev_info(dev,
5055                          "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
5056                          ch_count,
5057                          ud->bchan_cnt - bitmap_weight(ud->bchan_map,
5058                                                        ud->bchan_cnt),
5059                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5060                                                        ud->tchan_cnt),
5061                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5062                                                        ud->rchan_cnt));
5063                 break;
5064         case DMA_TYPE_PKTDMA:
5065                 dev_info(dev,
5066                          "Channels: %d (tchan: %u, rchan: %u)\n",
5067                          ch_count,
5068                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5069                                                        ud->tchan_cnt),
5070                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5071                                                        ud->rchan_cnt));
5072                 break;
5073         default:
5074                 break;
5075         }
5076
5077         return ch_count;
5078 }
5079
5080 static int udma_setup_rx_flush(struct udma_dev *ud)
5081 {
5082         struct udma_rx_flush *rx_flush = &ud->rx_flush;
5083         struct cppi5_desc_hdr_t *tr_desc;
5084         struct cppi5_tr_type1_t *tr_req;
5085         struct cppi5_host_desc_t *desc;
5086         struct device *dev = ud->dev;
5087         struct udma_hwdesc *hwdesc;
5088         size_t tr_size;
5089
5090         /* Allocate 1K buffer for discarded data on RX channel teardown */
5091         rx_flush->buffer_size = SZ_1K;
5092         rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
5093                                               GFP_KERNEL);
5094         if (!rx_flush->buffer_vaddr)
5095                 return -ENOMEM;
5096
5097         rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
5098                                                 rx_flush->buffer_size,
5099                                                 DMA_TO_DEVICE);
5100         if (dma_mapping_error(dev, rx_flush->buffer_paddr))
5101                 return -ENOMEM;
5102
5103         /* Set up descriptor to be used for TR mode */
5104         hwdesc = &rx_flush->hwdescs[0];
5105         tr_size = sizeof(struct cppi5_tr_type1_t);
5106         hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
5107         hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
5108                                         ud->desc_align);
5109
5110         hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5111                                                 GFP_KERNEL);
5112         if (!hwdesc->cppi5_desc_vaddr)
5113                 return -ENOMEM;
5114
5115         hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5116                                                   hwdesc->cppi5_desc_size,
5117                                                   DMA_TO_DEVICE);
5118         if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5119                 return -ENOMEM;
5120
5121         /* Start of the TR req records */
5122         hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
5123         /* Start address of the TR response array */
5124         hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
5125
5126         tr_desc = hwdesc->cppi5_desc_vaddr;
5127         cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
5128         cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5129         cppi5_desc_set_retpolicy(tr_desc, 0, 0);
5130
5131         tr_req = hwdesc->tr_req_base;
5132         cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
5133                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
5134         cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
5135
5136         tr_req->addr = rx_flush->buffer_paddr;
5137         tr_req->icnt0 = rx_flush->buffer_size;
5138         tr_req->icnt1 = 1;
5139
5140         dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5141                                    hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5142
5143         /* Set up descriptor to be used for packet mode */
5144         hwdesc = &rx_flush->hwdescs[1];
5145         hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
5146                                         CPPI5_INFO0_HDESC_EPIB_SIZE +
5147                                         CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
5148                                         ud->desc_align);
5149
5150         hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5151                                                 GFP_KERNEL);
5152         if (!hwdesc->cppi5_desc_vaddr)
5153                 return -ENOMEM;
5154
5155         hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5156                                                   hwdesc->cppi5_desc_size,
5157                                                   DMA_TO_DEVICE);
5158         if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5159                 return -ENOMEM;
5160
5161         desc = hwdesc->cppi5_desc_vaddr;
5162         cppi5_hdesc_init(desc, 0, 0);
5163         cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5164         cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
5165
5166         cppi5_hdesc_attach_buf(desc,
5167                                rx_flush->buffer_paddr, rx_flush->buffer_size,
5168                                rx_flush->buffer_paddr, rx_flush->buffer_size);
5169
5170         dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5171                                    hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5172         return 0;
5173 }
5174
5175 #ifdef CONFIG_DEBUG_FS
5176 static void udma_dbg_summary_show_chan(struct seq_file *s,
5177                                        struct dma_chan *chan)
5178 {
5179         struct udma_chan *uc = to_udma_chan(chan);
5180         struct udma_chan_config *ucc = &uc->config;
5181
5182         seq_printf(s, " %-13s| %s", dma_chan_name(chan),
5183                    chan->dbg_client_name ?: "in-use");
5184         if (ucc->tr_trigger_type)
5185                 seq_puts(s, " (triggered, ");
5186         else
5187                 seq_printf(s, " (%s, ",
5188                            dmaengine_get_direction_text(uc->config.dir));
5189
5190         switch (uc->config.dir) {
5191         case DMA_MEM_TO_MEM:
5192                 if (uc->ud->match_data->type == DMA_TYPE_BCDMA) {
5193                         seq_printf(s, "bchan%d)\n", uc->bchan->id);
5194                         return;
5195                 }
5196
5197                 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
5198                            ucc->src_thread, ucc->dst_thread);
5199                 break;
5200         case DMA_DEV_TO_MEM:
5201                 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
5202                            ucc->src_thread, ucc->dst_thread);
5203                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5204                         seq_printf(s, "rflow%d, ", uc->rflow->id);
5205                 break;
5206         case DMA_MEM_TO_DEV:
5207                 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
5208                            ucc->src_thread, ucc->dst_thread);
5209                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5210                         seq_printf(s, "tflow%d, ", uc->tchan->tflow_id);
5211                 break;
5212         default:
5213                 seq_printf(s, ")\n");
5214                 return;
5215         }
5216
5217         if (ucc->ep_type == PSIL_EP_NATIVE) {
5218                 seq_printf(s, "PSI-L Native");
5219                 if (ucc->metadata_size) {
5220                         seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
5221                         if (ucc->psd_size)
5222                                 seq_printf(s, " PSDsize:%u", ucc->psd_size);
5223                         seq_printf(s, " ]");
5224                 }
5225         } else {
5226                 seq_printf(s, "PDMA");
5227                 if (ucc->enable_acc32 || ucc->enable_burst)
5228                         seq_printf(s, "[%s%s ]",
5229                                    ucc->enable_acc32 ? " ACC32" : "",
5230                                    ucc->enable_burst ? " BURST" : "");
5231         }
5232
5233         seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
5234 }
5235
5236 static void udma_dbg_summary_show(struct seq_file *s,
5237                                   struct dma_device *dma_dev)
5238 {
5239         struct dma_chan *chan;
5240
5241         list_for_each_entry(chan, &dma_dev->channels, device_node) {
5242                 if (chan->client_count)
5243                         udma_dbg_summary_show_chan(s, chan);
5244         }
5245 }
5246 #endif /* CONFIG_DEBUG_FS */
5247
5248 static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud)
5249 {
5250         const struct udma_match_data *match_data = ud->match_data;
5251         u8 tpl;
5252
5253         if (!match_data->enable_memcpy_support)
5254                 return DMAENGINE_ALIGN_8_BYTES;
5255
5256         /* Get the highest TPL level the device supports for memcpy */
5257         if (ud->bchan_cnt)
5258                 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0);
5259         else if (ud->tchan_cnt)
5260                 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0);
5261         else
5262                 return DMAENGINE_ALIGN_8_BYTES;
5263
5264         switch (match_data->burst_size[tpl]) {
5265         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES:
5266                 return DMAENGINE_ALIGN_256_BYTES;
5267         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES:
5268                 return DMAENGINE_ALIGN_128_BYTES;
5269         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES:
5270         fallthrough;
5271         default:
5272                 return DMAENGINE_ALIGN_64_BYTES;
5273         }
5274 }
5275
5276 #define TI_UDMAC_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
5277                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
5278                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
5279                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
5280                                  BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
5281
5282 static int udma_probe(struct platform_device *pdev)
5283 {
5284         struct device_node *navss_node = pdev->dev.parent->of_node;
5285         const struct soc_device_attribute *soc;
5286         struct device *dev = &pdev->dev;
5287         struct udma_dev *ud;
5288         const struct of_device_id *match;
5289         int i, ret;
5290         int ch_count;
5291
5292         ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
5293         if (ret)
5294                 dev_err(dev, "failed to set dma mask stuff\n");
5295
5296         ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
5297         if (!ud)
5298                 return -ENOMEM;
5299
5300         match = of_match_node(udma_of_match, dev->of_node);
5301         if (!match) {
5302                 dev_err(dev, "No compatible match found\n");
5303                 return -ENODEV;
5304         }
5305         ud->match_data = match->data;
5306
5307         ud->soc_data = ud->match_data->soc_data;
5308         if (!ud->soc_data) {
5309                 soc = soc_device_match(k3_soc_devices);
5310                 if (!soc) {
5311                         dev_err(dev, "No compatible SoC found\n");
5312                         return -ENODEV;
5313                 }
5314                 ud->soc_data = soc->data;
5315         }
5316
5317         ret = udma_get_mmrs(pdev, ud);
5318         if (ret)
5319                 return ret;
5320
5321         ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
5322         if (IS_ERR(ud->tisci_rm.tisci))
5323                 return PTR_ERR(ud->tisci_rm.tisci);
5324
5325         ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
5326                                    &ud->tisci_rm.tisci_dev_id);
5327         if (ret) {
5328                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
5329                 return ret;
5330         }
5331         pdev->id = ud->tisci_rm.tisci_dev_id;
5332
5333         ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
5334                                    &ud->tisci_rm.tisci_navss_dev_id);
5335         if (ret) {
5336                 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
5337                 return ret;
5338         }
5339
5340         if (ud->match_data->type == DMA_TYPE_UDMA) {
5341                 ret = of_property_read_u32(dev->of_node, "ti,udma-atype",
5342                                            &ud->atype);
5343                 if (!ret && ud->atype > 2) {
5344                         dev_err(dev, "Invalid atype: %u\n", ud->atype);
5345                         return -EINVAL;
5346                 }
5347         } else {
5348                 ret = of_property_read_u32(dev->of_node, "ti,asel",
5349                                            &ud->asel);
5350                 if (!ret && ud->asel > 15) {
5351                         dev_err(dev, "Invalid asel: %u\n", ud->asel);
5352                         return -EINVAL;
5353                 }
5354         }
5355
5356         ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
5357         ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
5358
5359         if (ud->match_data->type == DMA_TYPE_UDMA) {
5360                 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
5361         } else {
5362                 struct k3_ringacc_init_data ring_init_data;
5363
5364                 ring_init_data.tisci = ud->tisci_rm.tisci;
5365                 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
5366                 if (ud->match_data->type == DMA_TYPE_BCDMA) {
5367                         ring_init_data.num_rings = ud->bchan_cnt +
5368                                                    ud->tchan_cnt +
5369                                                    ud->rchan_cnt;
5370                 } else {
5371                         ring_init_data.num_rings = ud->rflow_cnt +
5372                                                    ud->tflow_cnt;
5373                 }
5374
5375                 ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data);
5376         }
5377
5378         if (IS_ERR(ud->ringacc))
5379                 return PTR_ERR(ud->ringacc);
5380
5381         dev->msi.domain = of_msi_get_domain(dev, dev->of_node,
5382                                             DOMAIN_BUS_TI_SCI_INTA_MSI);
5383         if (!dev->msi.domain) {
5384                 return -EPROBE_DEFER;
5385         }
5386
5387         dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
5388         /* cyclic operation is not supported via PKTDMA */
5389         if (ud->match_data->type != DMA_TYPE_PKTDMA) {
5390                 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
5391                 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
5392         }
5393
5394         ud->ddev.device_config = udma_slave_config;
5395         ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
5396         ud->ddev.device_issue_pending = udma_issue_pending;
5397         ud->ddev.device_tx_status = udma_tx_status;
5398         ud->ddev.device_pause = udma_pause;
5399         ud->ddev.device_resume = udma_resume;
5400         ud->ddev.device_terminate_all = udma_terminate_all;
5401         ud->ddev.device_synchronize = udma_synchronize;
5402 #ifdef CONFIG_DEBUG_FS
5403         ud->ddev.dbg_summary_show = udma_dbg_summary_show;
5404 #endif
5405
5406         switch (ud->match_data->type) {
5407         case DMA_TYPE_UDMA:
5408                 ud->ddev.device_alloc_chan_resources =
5409                                         udma_alloc_chan_resources;
5410                 break;
5411         case DMA_TYPE_BCDMA:
5412                 ud->ddev.device_alloc_chan_resources =
5413                                         bcdma_alloc_chan_resources;
5414                 ud->ddev.device_router_config = bcdma_router_config;
5415                 break;
5416         case DMA_TYPE_PKTDMA:
5417                 ud->ddev.device_alloc_chan_resources =
5418                                         pktdma_alloc_chan_resources;
5419                 break;
5420         default:
5421                 return -EINVAL;
5422         }
5423         ud->ddev.device_free_chan_resources = udma_free_chan_resources;
5424
5425         ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
5426         ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
5427         ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
5428         ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
5429         ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
5430                                        DESC_METADATA_ENGINE;
5431         if (ud->match_data->enable_memcpy_support &&
5432             !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) {
5433                 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
5434                 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
5435                 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
5436         }
5437
5438         ud->ddev.dev = dev;
5439         ud->dev = dev;
5440         ud->psil_base = ud->match_data->psil_base;
5441
5442         INIT_LIST_HEAD(&ud->ddev.channels);
5443         INIT_LIST_HEAD(&ud->desc_to_purge);
5444
5445         ch_count = setup_resources(ud);
5446         if (ch_count <= 0)
5447                 return ch_count;
5448
5449         spin_lock_init(&ud->lock);
5450         INIT_WORK(&ud->purge_work, udma_purge_desc_work);
5451
5452         ud->desc_align = 64;
5453         if (ud->desc_align < dma_get_cache_alignment())
5454                 ud->desc_align = dma_get_cache_alignment();
5455
5456         ret = udma_setup_rx_flush(ud);
5457         if (ret)
5458                 return ret;
5459
5460         for (i = 0; i < ud->bchan_cnt; i++) {
5461                 struct udma_bchan *bchan = &ud->bchans[i];
5462
5463                 bchan->id = i;
5464                 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
5465         }
5466
5467         for (i = 0; i < ud->tchan_cnt; i++) {
5468                 struct udma_tchan *tchan = &ud->tchans[i];
5469
5470                 tchan->id = i;
5471                 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
5472         }
5473
5474         for (i = 0; i < ud->rchan_cnt; i++) {
5475                 struct udma_rchan *rchan = &ud->rchans[i];
5476
5477                 rchan->id = i;
5478                 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
5479         }
5480
5481         for (i = 0; i < ud->rflow_cnt; i++) {
5482                 struct udma_rflow *rflow = &ud->rflows[i];
5483
5484                 rflow->id = i;
5485         }
5486
5487         for (i = 0; i < ch_count; i++) {
5488                 struct udma_chan *uc = &ud->channels[i];
5489
5490                 uc->ud = ud;
5491                 uc->vc.desc_free = udma_desc_free;
5492                 uc->id = i;
5493                 uc->bchan = NULL;
5494                 uc->tchan = NULL;
5495                 uc->rchan = NULL;
5496                 uc->config.remote_thread_id = -1;
5497                 uc->config.mapped_channel_id = -1;
5498                 uc->config.default_flow_id = -1;
5499                 uc->config.dir = DMA_MEM_TO_MEM;
5500                 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
5501                                           dev_name(dev), i);
5502
5503                 vchan_init(&uc->vc, &ud->ddev);
5504                 /* Use custom vchan completion handling */
5505                 tasklet_setup(&uc->vc.task, udma_vchan_complete);
5506                 init_completion(&uc->teardown_completed);
5507                 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
5508         }
5509
5510         /* Configure the copy_align to the maximum burst size the device supports */
5511         ud->ddev.copy_align = udma_get_copy_align(ud);
5512
5513         ret = dma_async_device_register(&ud->ddev);
5514         if (ret) {
5515                 dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
5516                 return ret;
5517         }
5518
5519         platform_set_drvdata(pdev, ud);
5520
5521         ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
5522         if (ret) {
5523                 dev_err(dev, "failed to register of_dma controller\n");
5524                 dma_async_device_unregister(&ud->ddev);
5525         }
5526
5527         return ret;
5528 }
5529
5530 static int __maybe_unused udma_pm_suspend(struct device *dev)
5531 {
5532         struct udma_dev *ud = dev_get_drvdata(dev);
5533         struct dma_device *dma_dev = &ud->ddev;
5534         struct dma_chan *chan;
5535         struct udma_chan *uc;
5536
5537         list_for_each_entry(chan, &dma_dev->channels, device_node) {
5538                 if (chan->client_count) {
5539                         uc = to_udma_chan(chan);
5540                         /* backup the channel configuration */
5541                         memcpy(&uc->backup_config, &uc->config,
5542                                sizeof(struct udma_chan_config));
5543                         dev_dbg(dev, "Suspending channel %s\n",
5544                                 dma_chan_name(chan));
5545                         ud->ddev.device_free_chan_resources(chan);
5546                 }
5547         }
5548
5549         return 0;
5550 }
5551
5552 static int __maybe_unused udma_pm_resume(struct device *dev)
5553 {
5554         struct udma_dev *ud = dev_get_drvdata(dev);
5555         struct dma_device *dma_dev = &ud->ddev;
5556         struct dma_chan *chan;
5557         struct udma_chan *uc;
5558         int ret;
5559
5560         list_for_each_entry(chan, &dma_dev->channels, device_node) {
5561                 if (chan->client_count) {
5562                         uc = to_udma_chan(chan);
5563                         /* restore the channel configuration */
5564                         memcpy(&uc->config, &uc->backup_config,
5565                                sizeof(struct udma_chan_config));
5566                         dev_dbg(dev, "Resuming channel %s\n",
5567                                 dma_chan_name(chan));
5568                         ret = ud->ddev.device_alloc_chan_resources(chan);
5569                         if (ret)
5570                                 return ret;
5571                 }
5572         }
5573
5574         return 0;
5575 }
5576
5577 static const struct dev_pm_ops udma_pm_ops = {
5578         SET_LATE_SYSTEM_SLEEP_PM_OPS(udma_pm_suspend, udma_pm_resume)
5579 };
5580
5581 static struct platform_driver udma_driver = {
5582         .driver = {
5583                 .name   = "ti-udma",
5584                 .of_match_table = udma_of_match,
5585                 .suppress_bind_attrs = true,
5586                 .pm = &udma_pm_ops,
5587         },
5588         .probe          = udma_probe,
5589 };
5590
5591 module_platform_driver(udma_driver);
5592 MODULE_LICENSE("GPL v2");
5593
5594 /* Private interfaces to UDMA */
5595 #include "k3-udma-private.c"