drivers/dma/ti/k3-udma-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 NAVSS DMA glue interface
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/atomic.h>
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/io.h>
  14 #include <linux/init.h>
  15 #include <linux/of.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/soc/ti/k3-ringacc.h>
  18 #include <linux/dma/ti-cppi5.h>
  19 #include <linux/dma/k3-udma-glue.h>
  20
  21 #include "k3-udma.h"
  22 #include "k3-psil-priv.h"
  23
  24 struct k3_udma_glue_common {
  25         struct device *dev;
  26         struct device chan_dev;
  27         struct udma_dev *udmax;
  28         const struct udma_tisci_rm *tisci_rm;
  29         struct k3_ringacc *ringacc;
  30         u32 src_thread;
  31         u32 dst_thread;
  32
  33         u32  hdesc_size;
  34         bool epib;
  35         u32  psdata_size;
  36         u32  swdata_size;
  37         u32  atype_asel;
  38         struct psil_endpoint_config *ep_config;
  39 };
  40
  41 struct k3_udma_glue_tx_channel {
  42         struct k3_udma_glue_common common;
  43
  44         struct udma_tchan *udma_tchanx;
  45         int udma_tchan_id;
  46
  47         struct k3_ring *ringtx;
  48         struct k3_ring *ringtxcq;
  49
  50         bool psil_paired;
  51
  52         int virq;
  53
  54         atomic_t free_pkts;
  55         bool tx_pause_on_err;
  56         bool tx_filt_einfo;
  57         bool tx_filt_pswords;
  58         bool tx_supr_tdpkt;
  59
  60         int udma_tflow_id;
  61 };
  62
  63 struct k3_udma_glue_rx_flow {
  64         struct udma_rflow *udma_rflow;
  65         int udma_rflow_id;
  66         struct k3_ring *ringrx;
  67         struct k3_ring *ringrxfdq;
  68
  69         int virq;
  70 };
  71
  72 struct k3_udma_glue_rx_channel {
  73         struct k3_udma_glue_common common;
  74
  75         struct udma_rchan *udma_rchanx;
  76         int udma_rchan_id;
  77         bool remote;
  78
  79         bool psil_paired;
  80
  81         u32  swdata_size;
  82         int  flow_id_base;
  83
  84         struct k3_udma_glue_rx_flow *flows;
  85         u32 flow_num;
  86         u32 flows_ready;
  87 };
  88
  89 static void k3_udma_chan_dev_release(struct device *dev)
  90 {
  91         /* The struct containing the device is devm managed */
  92 }
  93
  94 static struct class k3_udma_glue_devclass = {
  95         .name           = "k3_udma_glue_chan",
  96         .dev_release    = k3_udma_chan_dev_release,
  97 };
  98
  99 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000
 100
 101 static int of_k3_udma_glue_parse(struct device_node *udmax_np,
 102                                  struct k3_udma_glue_common *common)
 103 {
 104         common->udmax = of_xudma_dev_get(udmax_np, NULL);
 105         if (IS_ERR(common->udmax))
 106                 return PTR_ERR(common->udmax);
 107
 108         common->ringacc = xudma_get_ringacc(common->udmax);
 109         common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
 110
 111         return 0;
 112 }
 113
 114 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
 115                 const char *name, struct k3_udma_glue_common *common,
 116                 bool tx_chn)
 117 {
 118         struct of_phandle_args dma_spec;
 119         u32 thread_id;
 120         int ret = 0;
 121         int index;
 122
 123         if (unlikely(!name))
 124                 return -EINVAL;
 125
 126         index = of_property_match_string(chn_np, "dma-names", name);
 127         if (index < 0)
 128                 return index;
 129
 130         if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
 131                                        &dma_spec))
 132                 return -ENOENT;
 133
 134         ret = of_k3_udma_glue_parse(dma_spec.np, common);
 135         if (ret)
 136                 goto out_put_spec;
 137
 138         thread_id = dma_spec.args[0];
 139         if (dma_spec.args_count == 2) {
 140                 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
 141                         dev_err(common->dev, "Invalid channel atype: %u\n",
 142                                 dma_spec.args[1]);
 143                         ret = -EINVAL;
 144                         goto out_put_spec;
 145                 }
 146                 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
 147                         dev_err(common->dev, "Invalid channel asel: %u\n",
 148                                 dma_spec.args[1]);
 149                         ret = -EINVAL;
 150                         goto out_put_spec;
 151                 }
 152
 153                 common->atype_asel = dma_spec.args[1];
 154         }
 155
 156         if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 157                 ret = -EINVAL;
 158                 goto out_put_spec;
 159         }
 160
 161         if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 162                 ret = -EINVAL;
 163                 goto out_put_spec;
 164         }
 165
 166         /* get psil endpoint config */
 167         common->ep_config = psil_get_ep_config(thread_id);
 168         if (IS_ERR(common->ep_config)) {
 169                 dev_err(common->dev,
 170                         "No configuration for psi-l thread 0x%04x\n",
 171                         thread_id);
 172                 ret = PTR_ERR(common->ep_config);
 173                 goto out_put_spec;
 174         }
 175
 176         common->epib = common->ep_config->needs_epib;
 177         common->psdata_size = common->ep_config->psd_size;
 178
 179         if (tx_chn)
 180                 common->dst_thread = thread_id;
 181         else
 182                 common->src_thread = thread_id;
 183
 184 out_put_spec:
 185         of_node_put(dma_spec.np);
 186         return ret;
 187 };
 188
 189 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 190 {
 191         struct device *dev = tx_chn->common.dev;
 192
 193         dev_dbg(dev, "dump_tx_chn:\n"
 194                 "udma_tchan_id: %d\n"
 195                 "src_thread: %08x\n"
 196                 "dst_thread: %08x\n",
 197                 tx_chn->udma_tchan_id,
 198                 tx_chn->common.src_thread,
 199                 tx_chn->common.dst_thread);
 200 }
 201
 202 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
 203                                         char *mark)
 204 {
 205         struct device *dev = chn->common.dev;
 206
 207         dev_dbg(dev, "=== dump ===> %s\n", mark);
 208         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 209                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
 210         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 211                 xudma_tchanrt_read(chn->udma_tchanx,
 212                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 213         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 214                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
 215         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 216                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
 217         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 218                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
 219 }
 220
 221 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 222 {
 223         const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
 224         struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
 225
 226         memset(&req, 0, sizeof(req));
 227
 228         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
 229                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
 230                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
 231                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 232                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
 233                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 234                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 235                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 236         req.nav_id = tisci_rm->tisci_dev_id;
 237         req.index = tx_chn->udma_tchan_id;
 238         if (tx_chn->tx_pause_on_err)
 239                 req.tx_pause_on_err = 1;
 240         if (tx_chn->tx_filt_einfo)
 241                 req.tx_filt_einfo = 1;
 242         if (tx_chn->tx_filt_pswords)
 243                 req.tx_filt_pswords = 1;
 244         req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 245         if (tx_chn->tx_supr_tdpkt)
 246                 req.tx_supr_tdpkt = 1;
 247         req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
 248         req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 249         req.tx_atype = tx_chn->common.atype_asel;
 250
 251         return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
 252 }
 253
 254 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
 255                 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
 256 {
 257         struct k3_udma_glue_tx_channel *tx_chn;
 258         int ret;
 259
 260         tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
 261         if (!tx_chn)
 262                 return ERR_PTR(-ENOMEM);
 263
 264         tx_chn->common.dev = dev;
 265         tx_chn->common.swdata_size = cfg->swdata_size;
 266         tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
 267         tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
 268         tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
 269         tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
 270
 271         /* parse of udmap channel */
 272         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 273                                         &tx_chn->common, true);
 274         if (ret)
 275                 goto err;
 276
 277         tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
 278                                                 tx_chn->common.psdata_size,
 279                                                 tx_chn->common.swdata_size);
 280
 281         if (xudma_is_pktdma(tx_chn->common.udmax))
 282                 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
 283         else
 284                 tx_chn->udma_tchan_id = -1;
 285
 286         /* request and cfg UDMAP TX channel */
 287         tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
 288                                               tx_chn->udma_tchan_id);
 289         if (IS_ERR(tx_chn->udma_tchanx)) {
 290                 ret = PTR_ERR(tx_chn->udma_tchanx);
 291                 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
 292                 goto err;
 293         }
 294         tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
 295
 296         tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 297         tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
 298         dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
 299                      tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
 300         ret = device_register(&tx_chn->common.chan_dev);
 301         if (ret) {
 302                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 303                 put_device(&tx_chn->common.chan_dev);
 304                 tx_chn->common.chan_dev.parent = NULL;
 305                 goto err;
 306         }
 307
 308         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 309                 /* prepare the channel device as coherent */
 310                 tx_chn->common.chan_dev.dma_coherent = true;
 311                 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
 312                                              DMA_BIT_MASK(48));
 313         }
 314
 315         atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
 316
 317         if (xudma_is_pktdma(tx_chn->common.udmax))
 318                 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
 319         else
 320                 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
 321
 322         /* request and cfg rings */
 323         ret =  k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
 324                                              tx_chn->udma_tflow_id, -1,
 325                                              &tx_chn->ringtx,
 326                                              &tx_chn->ringtxcq);
 327         if (ret) {
 328                 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
 329                 goto err;
 330         }
 331
 332         /* Set the dma_dev for the rings to be configured */
 333         cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
 334         cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
 335
 336         /* Set the ASEL value for DMA rings of PKTDMA */
 337         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 338                 cfg->tx_cfg.asel = tx_chn->common.atype_asel;
 339                 cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
 340         }
 341
 342         ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
 343         if (ret) {
 344                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 345                 goto err;
 346         }
 347
 348         ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
 349         if (ret) {
 350                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 351                 goto err;
 352         }
 353
 354         /* request and cfg psi-l */
 355         tx_chn->common.src_thread =
 356                         xudma_dev_get_psil_base(tx_chn->common.udmax) +
 357                         tx_chn->udma_tchan_id;
 358
 359         ret = k3_udma_glue_cfg_tx_chn(tx_chn);
 360         if (ret) {
 361                 dev_err(dev, "Failed to cfg tchan %d\n", ret);
 362                 goto err;
 363         }
 364
 365         k3_udma_glue_dump_tx_chn(tx_chn);
 366
 367         return tx_chn;
 368
 369 err:
 370         k3_udma_glue_release_tx_chn(tx_chn);
 371         return ERR_PTR(ret);
 372 }
 373 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
 374
 375 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 376 {
 377         if (tx_chn->psil_paired) {
 378                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 379                                         tx_chn->common.src_thread,
 380                                         tx_chn->common.dst_thread);
 381                 tx_chn->psil_paired = false;
 382         }
 383
 384         if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
 385                 xudma_tchan_put(tx_chn->common.udmax,
 386                                 tx_chn->udma_tchanx);
 387
 388         if (tx_chn->ringtxcq)
 389                 k3_ringacc_ring_free(tx_chn->ringtxcq);
 390
 391         if (tx_chn->ringtx)
 392                 k3_ringacc_ring_free(tx_chn->ringtx);
 393
 394         if (tx_chn->common.chan_dev.parent) {
 395                 device_unregister(&tx_chn->common.chan_dev);
 396                 tx_chn->common.chan_dev.parent = NULL;
 397         }
 398 }
 399 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
 400
 401 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 402                              struct cppi5_host_desc_t *desc_tx,
 403                              dma_addr_t desc_dma)
 404 {
 405         u32 ringtxcq_id;
 406
 407         if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
 408                 return -ENOMEM;
 409
 410         ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 411         cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
 412
 413         return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
 414 }
 415 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
 416
 417 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 418                             dma_addr_t *desc_dma)
 419 {
 420         int ret;
 421
 422         ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
 423         if (!ret)
 424                 atomic_inc(&tx_chn->free_pkts);
 425
 426         return ret;
 427 }
 428 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
 429
 430 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 431 {
 432         int ret;
 433
 434         ret = xudma_navss_psil_pair(tx_chn->common.udmax,
 435                                     tx_chn->common.src_thread,
 436                                     tx_chn->common.dst_thread);
 437         if (ret) {
 438                 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
 439                 return ret;
 440         }
 441
 442         tx_chn->psil_paired = true;
 443
 444         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
 445                             UDMA_PEER_RT_EN_ENABLE);
 446
 447         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 448                             UDMA_CHAN_RT_CTL_EN);
 449
 450         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
 451         return 0;
 452 }
 453 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
 454
 455 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 456 {
 457         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
 458
 459         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
 460
 461         xudma_tchanrt_write(tx_chn->udma_tchanx,
 462                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 463         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
 464
 465         if (tx_chn->psil_paired) {
 466                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 467                                         tx_chn->common.src_thread,
 468                                         tx_chn->common.dst_thread);
 469                 tx_chn->psil_paired = false;
 470         }
 471 }
 472 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
 473
 474 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 475                                bool sync)
 476 {
 477         int i = 0;
 478         u32 val;
 479
 480         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
 481
 482         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 483                             UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
 484
 485         val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
 486
 487         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
 488                 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 489                                          UDMA_CHAN_RT_CTL_REG);
 490                 udelay(1);
 491                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
 492                         dev_err(tx_chn->common.dev, "TX tdown timeout\n");
 493                         break;
 494                 }
 495                 i++;
 496         }
 497
 498         val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 499                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
 500         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
 501                 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
 502         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
 503 }
 504 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
 505
 506 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 507                                void *data,
 508                                void (*cleanup)(void *data, dma_addr_t desc_dma))
 509 {
 510         struct device *dev = tx_chn->common.dev;
 511         dma_addr_t desc_dma;
 512         int occ_tx, i, ret;
 513
 514         /*
 515          * TXQ reset need to be special way as it is input for udma and its
 516          * state cached by udma, so:
 517          * 1) save TXQ occ
 518          * 2) clean up TXQ and call callback .cleanup() for each desc
 519          * 3) reset TXQ in a special way
 520          */
 521         occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
 522         dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
 523
 524         for (i = 0; i < occ_tx; i++) {
 525                 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
 526                 if (ret) {
 527                         if (ret != -ENODATA)
 528                                 dev_err(dev, "TX reset pop %d\n", ret);
 529                         break;
 530                 }
 531                 cleanup(data, desc_dma);
 532         }
 533
 534         /* reset TXCQ as it is not input for udma - expected to be empty */
 535         k3_ringacc_ring_reset(tx_chn->ringtxcq);
 536         k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
 537 }
 538 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
 539
 540 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
 541 {
 542         return tx_chn->common.hdesc_size;
 543 }
 544 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
 545
 546 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
 547 {
 548         return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 549 }
 550 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
 551
 552 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
 553 {
 554         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 555                 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
 556                                                           tx_chn->udma_tflow_id);
 557         } else {
 558                 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
 559         }
 560
 561         return tx_chn->virq;
 562 }
 563 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
 564
 565 struct device *
 566         k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
 567 {
 568         if (xudma_is_pktdma(tx_chn->common.udmax) &&
 569             (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
 570                 return &tx_chn->common.chan_dev;
 571
 572         return xudma_get_device(tx_chn->common.udmax);
 573 }
 574 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
 575
 576 void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
 577                                        dma_addr_t *addr)
 578 {
 579         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 580             !tx_chn->common.atype_asel)
 581                 return;
 582
 583         *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
 584 }
 585 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
 586
 587 void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
 588                                        dma_addr_t *addr)
 589 {
 590         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 591             !tx_chn->common.atype_asel)
 592                 return;
 593
 594         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
 595 }
 596 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
 597
 598 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 599 {
 600         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 601         struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
 602         int ret;
 603
 604         memset(&req, 0, sizeof(req));
 605
 606         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 607                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 608                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 609                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 610
 611         req.nav_id = tisci_rm->tisci_dev_id;
 612         req.index = rx_chn->udma_rchan_id;
 613         req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
 614         /*
 615          * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
 616          * and udmax impl, so just configure it to invalid value.
 617          * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
 618          */
 619         req.rxcq_qnum = 0xFFFF;
 620         if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
 621             rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
 622                 /* Default flow + extra ones */
 623                 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
 624                                     TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
 625                 req.flowid_start = rx_chn->flow_id_base;
 626                 req.flowid_cnt = rx_chn->flow_num;
 627         }
 628         req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 629         req.rx_atype = rx_chn->common.atype_asel;
 630
 631         ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
 632         if (ret)
 633                 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
 634                         rx_chn->udma_rchan_id, ret);
 635
 636         return ret;
 637 }
 638
 639 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 640                                          u32 flow_num)
 641 {
 642         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
 643
 644         if (IS_ERR_OR_NULL(flow->udma_rflow))
 645                 return;
 646
 647         if (flow->ringrxfdq)
 648                 k3_ringacc_ring_free(flow->ringrxfdq);
 649
 650         if (flow->ringrx)
 651                 k3_ringacc_ring_free(flow->ringrx);
 652
 653         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 654         flow->udma_rflow = NULL;
 655         rx_chn->flows_ready--;
 656 }
 657
 658 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 659                                     u32 flow_idx,
 660                                     struct k3_udma_glue_rx_flow_cfg *flow_cfg)
 661 {
 662         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
 663         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 664         struct device *dev = rx_chn->common.dev;
 665         struct ti_sci_msg_rm_udmap_flow_cfg req;
 666         int rx_ring_id;
 667         int rx_ringfdq_id;
 668         int ret = 0;
 669
 670         flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
 671                                            flow->udma_rflow_id);
 672         if (IS_ERR(flow->udma_rflow)) {
 673                 ret = PTR_ERR(flow->udma_rflow);
 674                 dev_err(dev, "UDMAX rflow get err %d\n", ret);
 675                 return ret;
 676         }
 677
 678         if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
 679                 ret = -ENODEV;
 680                 goto err_rflow_put;
 681         }
 682
 683         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 684                 rx_ringfdq_id = flow->udma_rflow_id +
 685                                 xudma_get_rflow_ring_offset(rx_chn->common.udmax);
 686                 rx_ring_id = 0;
 687         } else {
 688                 rx_ring_id = flow_cfg->ring_rxq_id;
 689                 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
 690         }
 691
 692         /* request and cfg rings */
 693         ret =  k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
 694                                              rx_ringfdq_id, rx_ring_id,
 695                                              &flow->ringrxfdq,
 696                                              &flow->ringrx);
 697         if (ret) {
 698                 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
 699                 goto err_rflow_put;
 700         }
 701
 702         /* Set the dma_dev for the rings to be configured */
 703         flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
 704         flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
 705
 706         /* Set the ASEL value for DMA rings of PKTDMA */
 707         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 708                 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
 709                 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
 710         }
 711
 712         ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
 713         if (ret) {
 714                 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
 715                 goto err_ringrxfdq_free;
 716         }
 717
 718         ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
 719         if (ret) {
 720                 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
 721                 goto err_ringrxfdq_free;
 722         }
 723
 724         if (rx_chn->remote) {
 725                 rx_ring_id = TI_SCI_RESOURCE_NULL;
 726                 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
 727         } else {
 728                 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
 729                 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
 730         }
 731
 732         memset(&req, 0, sizeof(req));
 733
 734         req.valid_params =
 735                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
 736                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
 737                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
 738                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
 739                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
 740                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
 741                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
 742                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
 743                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
 744                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
 745                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
 746                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
 747                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
 748         req.nav_id = tisci_rm->tisci_dev_id;
 749         req.flow_index = flow->udma_rflow_id;
 750         if (rx_chn->common.epib)
 751                 req.rx_einfo_present = 1;
 752         if (rx_chn->common.psdata_size)
 753                 req.rx_psinfo_present = 1;
 754         if (flow_cfg->rx_error_handling)
 755                 req.rx_error_handling = 1;
 756         req.rx_desc_type = 0;
 757         req.rx_dest_qnum = rx_ring_id;
 758         req.rx_src_tag_hi_sel = 0;
 759         req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
 760         req.rx_dest_tag_hi_sel = 0;
 761         req.rx_dest_tag_lo_sel = 0;
 762         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
 763         req.rx_fdq1_qnum = rx_ringfdq_id;
 764         req.rx_fdq2_qnum = rx_ringfdq_id;
 765         req.rx_fdq3_qnum = rx_ringfdq_id;
 766
 767         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
 768         if (ret) {
 769                 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
 770                         ret);
 771                 goto err_ringrxfdq_free;
 772         }
 773
 774         rx_chn->flows_ready++;
 775         dev_dbg(dev, "flow%d config done. ready:%d\n",
 776                 flow->udma_rflow_id, rx_chn->flows_ready);
 777
 778         return 0;
 779
 780 err_ringrxfdq_free:
 781         k3_ringacc_ring_free(flow->ringrxfdq);
 782         k3_ringacc_ring_free(flow->ringrx);
 783
 784 err_rflow_put:
 785         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 786         flow->udma_rflow = NULL;
 787
 788         return ret;
 789 }
 790
 791 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
 792 {
 793         struct device *dev = chn->common.dev;
 794
 795         dev_dbg(dev, "dump_rx_chn:\n"
 796                 "udma_rchan_id: %d\n"
 797                 "src_thread: %08x\n"
 798                 "dst_thread: %08x\n"
 799                 "epib: %d\n"
 800                 "hdesc_size: %u\n"
 801                 "psdata_size: %u\n"
 802                 "swdata_size: %u\n"
 803                 "flow_id_base: %d\n"
 804                 "flow_num: %d\n",
 805                 chn->udma_rchan_id,
 806                 chn->common.src_thread,
 807                 chn->common.dst_thread,
 808                 chn->common.epib,
 809                 chn->common.hdesc_size,
 810                 chn->common.psdata_size,
 811                 chn->common.swdata_size,
 812                 chn->flow_id_base,
 813                 chn->flow_num);
 814 }
 815
 816 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
 817                                         char *mark)
 818 {
 819         struct device *dev = chn->common.dev;
 820
 821         dev_dbg(dev, "=== dump ===> %s\n", mark);
 822
 823         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 824                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
 825         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 826                 xudma_rchanrt_read(chn->udma_rchanx,
 827                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 828         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 829                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
 830         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 831                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
 832         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 833                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
 834 }
 835
 836 static int
 837 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
 838                                struct k3_udma_glue_rx_channel_cfg *cfg)
 839 {
 840         int ret;
 841
 842         /* default rflow */
 843         if (cfg->flow_id_use_rxchan_id)
 844                 return 0;
 845
 846         /* not a GP rflows */
 847         if (rx_chn->flow_id_base != -1 &&
 848             !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 849                 return 0;
 850
 851         /* Allocate range of GP rflows */
 852         ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
 853                                          rx_chn->flow_id_base,
 854                                          rx_chn->flow_num);
 855         if (ret < 0) {
 856                 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
 857                         rx_chn->flow_id_base, rx_chn->flow_num, ret);
 858                 return ret;
 859         }
 860         rx_chn->flow_id_base = ret;
 861
 862         return 0;
 863 }
 864
 865 static struct k3_udma_glue_rx_channel *
 866 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
 867                                  struct k3_udma_glue_rx_channel_cfg *cfg)
 868 {
 869         struct k3_udma_glue_rx_channel *rx_chn;
 870         struct psil_endpoint_config *ep_cfg;
 871         int ret, i;
 872
 873         if (cfg->flow_id_num <= 0)
 874                 return ERR_PTR(-EINVAL);
 875
 876         if (cfg->flow_id_num != 1 &&
 877             (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
 878                 return ERR_PTR(-EINVAL);
 879
 880         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
 881         if (!rx_chn)
 882                 return ERR_PTR(-ENOMEM);
 883
 884         rx_chn->common.dev = dev;
 885         rx_chn->common.swdata_size = cfg->swdata_size;
 886         rx_chn->remote = false;
 887
 888         /* parse of udmap channel */
 889         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 890                                         &rx_chn->common, false);
 891         if (ret)
 892                 goto err;
 893
 894         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
 895                                                 rx_chn->common.psdata_size,
 896                                                 rx_chn->common.swdata_size);
 897
 898         ep_cfg = rx_chn->common.ep_config;
 899
 900         if (xudma_is_pktdma(rx_chn->common.udmax))
 901                 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
 902         else
 903                 rx_chn->udma_rchan_id = -1;
 904
 905         /* request and cfg UDMAP RX channel */
 906         rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
 907                                               rx_chn->udma_rchan_id);
 908         if (IS_ERR(rx_chn->udma_rchanx)) {
 909                 ret = PTR_ERR(rx_chn->udma_rchanx);
 910                 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
 911                 goto err;
 912         }
 913         rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
 914
 915         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 916         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
 917         dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
 918                      rx_chn->udma_rchan_id, rx_chn->common.src_thread);
 919         ret = device_register(&rx_chn->common.chan_dev);
 920         if (ret) {
 921                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 922                 put_device(&rx_chn->common.chan_dev);
 923                 rx_chn->common.chan_dev.parent = NULL;
 924                 goto err;
 925         }
 926
 927         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 928                 /* prepare the channel device as coherent */
 929                 rx_chn->common.chan_dev.dma_coherent = true;
 930                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
 931                                              DMA_BIT_MASK(48));
 932         }
 933
 934         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 935                 int flow_start = cfg->flow_id_base;
 936                 int flow_end;
 937
 938                 if (flow_start == -1)
 939                         flow_start = ep_cfg->flow_start;
 940
 941                 flow_end = flow_start + cfg->flow_id_num - 1;
 942                 if (flow_start < ep_cfg->flow_start ||
 943                     flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
 944                         dev_err(dev, "Invalid flow range requested\n");
 945                         ret = -EINVAL;
 946                         goto err;
 947                 }
 948                 rx_chn->flow_id_base = flow_start;
 949         } else {
 950                 rx_chn->flow_id_base = cfg->flow_id_base;
 951
 952                 /* Use RX channel id as flow id: target dev can't generate flow_id */
 953                 if (cfg->flow_id_use_rxchan_id)
 954                         rx_chn->flow_id_base = rx_chn->udma_rchan_id;
 955         }
 956
 957         rx_chn->flow_num = cfg->flow_id_num;
 958
 959         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
 960                                      sizeof(*rx_chn->flows), GFP_KERNEL);
 961         if (!rx_chn->flows) {
 962                 ret = -ENOMEM;
 963                 goto err;
 964         }
 965
 966         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
 967         if (ret)
 968                 goto err;
 969
 970         for (i = 0; i < rx_chn->flow_num; i++)
 971                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
 972
 973         /* request and cfg psi-l */
 974         rx_chn->common.dst_thread =
 975                         xudma_dev_get_psil_base(rx_chn->common.udmax) +
 976                         rx_chn->udma_rchan_id;
 977
 978         ret = k3_udma_glue_cfg_rx_chn(rx_chn);
 979         if (ret) {
 980                 dev_err(dev, "Failed to cfg rchan %d\n", ret);
 981                 goto err;
 982         }
 983
 984         /* init default RX flow only if flow_num = 1 */
 985         if (cfg->def_flow_cfg) {
 986                 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
 987                 if (ret)
 988                         goto err;
 989         }
 990
 991         k3_udma_glue_dump_rx_chn(rx_chn);
 992
 993         return rx_chn;
 994
 995 err:
 996         k3_udma_glue_release_rx_chn(rx_chn);
 997         return ERR_PTR(ret);
 998 }
 999
1000 static struct k3_udma_glue_rx_channel *
1001 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
1002                                    struct k3_udma_glue_rx_channel_cfg *cfg)
1003 {
1004         struct k3_udma_glue_rx_channel *rx_chn;
1005         int ret, i;
1006
1007         if (cfg->flow_id_num <= 0 ||
1008             cfg->flow_id_use_rxchan_id ||
1009             cfg->def_flow_cfg ||
1010             cfg->flow_id_base < 0)
1011                 return ERR_PTR(-EINVAL);
1012
1013         /*
1014          * Remote RX channel is under control of Remote CPU core, so
1015          * Linux can only request and manipulate by dedicated RX flows
1016          */
1017
1018         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1019         if (!rx_chn)
1020                 return ERR_PTR(-ENOMEM);
1021
1022         rx_chn->common.dev = dev;
1023         rx_chn->common.swdata_size = cfg->swdata_size;
1024         rx_chn->remote = true;
1025         rx_chn->udma_rchan_id = -1;
1026         rx_chn->flow_num = cfg->flow_id_num;
1027         rx_chn->flow_id_base = cfg->flow_id_base;
1028         rx_chn->psil_paired = false;
1029
1030         /* parse of udmap channel */
1031         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1032                                         &rx_chn->common, false);
1033         if (ret)
1034                 goto err;
1035
1036         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1037                                                 rx_chn->common.psdata_size,
1038                                                 rx_chn->common.swdata_size);
1039
1040         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1041                                      sizeof(*rx_chn->flows), GFP_KERNEL);
1042         if (!rx_chn->flows) {
1043                 ret = -ENOMEM;
1044                 goto err;
1045         }
1046
1047         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1048         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1049         dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1050                      rx_chn->common.src_thread);
1051         ret = device_register(&rx_chn->common.chan_dev);
1052         if (ret) {
1053                 dev_err(dev, "Channel Device registration failed %d\n", ret);
1054                 put_device(&rx_chn->common.chan_dev);
1055                 rx_chn->common.chan_dev.parent = NULL;
1056                 goto err;
1057         }
1058
1059         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1060                 /* prepare the channel device as coherent */
1061                 rx_chn->common.chan_dev.dma_coherent = true;
1062                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1063                                              DMA_BIT_MASK(48));
1064         }
1065
1066         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1067         if (ret)
1068                 goto err;
1069
1070         for (i = 0; i < rx_chn->flow_num; i++)
1071                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1072
1073         k3_udma_glue_dump_rx_chn(rx_chn);
1074
1075         return rx_chn;
1076
1077 err:
1078         k3_udma_glue_release_rx_chn(rx_chn);
1079         return ERR_PTR(ret);
1080 }
1081
1082 struct k3_udma_glue_rx_channel *
1083 k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1084                             struct k3_udma_glue_rx_channel_cfg *cfg)
1085 {
1086         if (cfg->remote)
1087                 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1088         else
1089                 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1090 }
1091 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1092
1093 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1094 {
1095         int i;
1096
1097         if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1098                 return;
1099
1100         if (rx_chn->psil_paired) {
1101                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1102                                         rx_chn->common.src_thread,
1103                                         rx_chn->common.dst_thread);
1104                 rx_chn->psil_paired = false;
1105         }
1106
1107         for (i = 0; i < rx_chn->flow_num; i++)
1108                 k3_udma_glue_release_rx_flow(rx_chn, i);
1109
1110         if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1111                 xudma_free_gp_rflow_range(rx_chn->common.udmax,
1112                                           rx_chn->flow_id_base,
1113                                           rx_chn->flow_num);
1114
1115         if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1116                 xudma_rchan_put(rx_chn->common.udmax,
1117                                 rx_chn->udma_rchanx);
1118
1119         if (rx_chn->common.chan_dev.parent) {
1120                 device_unregister(&rx_chn->common.chan_dev);
1121                 rx_chn->common.chan_dev.parent = NULL;
1122         }
1123 }
1124 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1125
1126 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1127                               u32 flow_idx,
1128                               struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1129 {
1130         if (flow_idx >= rx_chn->flow_num)
1131                 return -EINVAL;
1132
1133         return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1134 }
1135 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1136
1137 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1138                                     u32 flow_idx)
1139 {
1140         struct k3_udma_glue_rx_flow *flow;
1141
1142         if (flow_idx >= rx_chn->flow_num)
1143                 return -EINVAL;
1144
1145         flow = &rx_chn->flows[flow_idx];
1146
1147         return k3_ringacc_get_ring_id(flow->ringrxfdq);
1148 }
1149 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1150
1151 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1152 {
1153         return rx_chn->flow_id_base;
1154 }
1155 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1156
1157 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1158                                 u32 flow_idx)
1159 {
1160         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1161         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1162         struct device *dev = rx_chn->common.dev;
1163         struct ti_sci_msg_rm_udmap_flow_cfg req;
1164         int rx_ring_id;
1165         int rx_ringfdq_id;
1166         int ret = 0;
1167
1168         if (!rx_chn->remote)
1169                 return -EINVAL;
1170
1171         rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1172         rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1173
1174         memset(&req, 0, sizeof(req));
1175
1176         req.valid_params =
1177                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1178                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1179                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1180                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1181                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1182         req.nav_id = tisci_rm->tisci_dev_id;
1183         req.flow_index = flow->udma_rflow_id;
1184         req.rx_dest_qnum = rx_ring_id;
1185         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1186         req.rx_fdq1_qnum = rx_ringfdq_id;
1187         req.rx_fdq2_qnum = rx_ringfdq_id;
1188         req.rx_fdq3_qnum = rx_ringfdq_id;
1189
1190         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1191         if (ret) {
1192                 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1193                         ret);
1194         }
1195
1196         return ret;
1197 }
1198 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1199
1200 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1201                                  u32 flow_idx)
1202 {
1203         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1204         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1205         struct device *dev = rx_chn->common.dev;
1206         struct ti_sci_msg_rm_udmap_flow_cfg req;
1207         int ret = 0;
1208
1209         if (!rx_chn->remote)
1210                 return -EINVAL;
1211
1212         memset(&req, 0, sizeof(req));
1213         req.valid_params =
1214                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1215                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1216                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1217                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1218                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1219         req.nav_id = tisci_rm->tisci_dev_id;
1220         req.flow_index = flow->udma_rflow_id;
1221         req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1222         req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1223         req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1224         req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1225         req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1226
1227         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1228         if (ret) {
1229                 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1230                         ret);
1231         }
1232
1233         return ret;
1234 }
1235 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1236
1237 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1238 {
1239         int ret;
1240
1241         if (rx_chn->remote)
1242                 return -EINVAL;
1243
1244         if (rx_chn->flows_ready < rx_chn->flow_num)
1245                 return -EINVAL;
1246
1247         ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1248                                     rx_chn->common.src_thread,
1249                                     rx_chn->common.dst_thread);
1250         if (ret) {
1251                 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1252                 return ret;
1253         }
1254
1255         rx_chn->psil_paired = true;
1256
1257         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
1258                             UDMA_CHAN_RT_CTL_EN);
1259
1260         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1261                             UDMA_PEER_RT_EN_ENABLE);
1262
1263         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1264         return 0;
1265 }
1266 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1267
1268 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1269 {
1270         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1271
1272         xudma_rchanrt_write(rx_chn->udma_rchanx,
1273                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1274         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
1275
1276         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1277
1278         if (rx_chn->psil_paired) {
1279                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1280                                         rx_chn->common.src_thread,
1281                                         rx_chn->common.dst_thread);
1282                 rx_chn->psil_paired = false;
1283         }
1284 }
1285 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1286
1287 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1288                                bool sync)
1289 {
1290         int i = 0;
1291         u32 val;
1292
1293         if (rx_chn->remote)
1294                 return;
1295
1296         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1297
1298         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1299                             UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1300
1301         val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
1302
1303         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1304                 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1305                                          UDMA_CHAN_RT_CTL_REG);
1306                 udelay(1);
1307                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1308                         dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1309                         break;
1310                 }
1311                 i++;
1312         }
1313
1314         val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1315                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
1316         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1317                 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1318         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1319 }
1320 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1321
1322 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1323                 u32 flow_num, void *data,
1324                 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1325 {
1326         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1327         struct device *dev = rx_chn->common.dev;
1328         dma_addr_t desc_dma;
1329         int occ_rx, i, ret;
1330
1331         /* reset RXCQ as it is not input for udma - expected to be empty */
1332         occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1333         dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1334
1335         /* Skip RX FDQ in case one FDQ is used for the set of flows */
1336         if (skip_fdq)
1337                 goto do_reset;
1338
1339         /*
1340          * RX FDQ reset need to be special way as it is input for udma and its
1341          * state cached by udma, so:
1342          * 1) save RX FDQ occ
1343          * 2) clean up RX FDQ and call callback .cleanup() for each desc
1344          * 3) reset RX FDQ in a special way
1345          */
1346         occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1347         dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1348
1349         for (i = 0; i < occ_rx; i++) {
1350                 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1351                 if (ret) {
1352                         if (ret != -ENODATA)
1353                                 dev_err(dev, "RX reset pop %d\n", ret);
1354                         break;
1355                 }
1356                 cleanup(data, desc_dma);
1357         }
1358
1359         k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1360
1361 do_reset:
1362         k3_ringacc_ring_reset(flow->ringrx);
1363 }
1364 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1365
1366 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1367                              u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1368                              dma_addr_t desc_dma)
1369 {
1370         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1371
1372         return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1373 }
1374 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1375
1376 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1377                             u32 flow_num, dma_addr_t *desc_dma)
1378 {
1379         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1380
1381         return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1382 }
1383 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1384
1385 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1386                             u32 flow_num)
1387 {
1388         struct k3_udma_glue_rx_flow *flow;
1389
1390         flow = &rx_chn->flows[flow_num];
1391
1392         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1393                 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1394                                                         flow->udma_rflow_id);
1395         } else {
1396                 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1397         }
1398
1399         return flow->virq;
1400 }
1401 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
1402
1403 struct device *
1404         k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1405 {
1406         if (xudma_is_pktdma(rx_chn->common.udmax) &&
1407             (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1408                 return &rx_chn->common.chan_dev;
1409
1410         return xudma_get_device(rx_chn->common.udmax);
1411 }
1412 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
1413
1414 void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1415                                        dma_addr_t *addr)
1416 {
1417         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1418             !rx_chn->common.atype_asel)
1419                 return;
1420
1421         *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1422 }
1423 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1424
1425 void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1426                                        dma_addr_t *addr)
1427 {
1428         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1429             !rx_chn->common.atype_asel)
1430                 return;
1431
1432         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1433 }
1434 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1435
1436 static int __init k3_udma_glue_class_init(void)
1437 {
1438         return class_register(&k3_udma_glue_devclass);
1439 }
1440
1441 module_init(k3_udma_glue_class_init);
1442 MODULE_LICENSE("GPL v2");