drivers/spi/spi-dw-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Special handling for DW DMA core
   4  *
   5  * Copyright (c) 2009, 2014 Intel Corporation.
   6  */
   7
   8 #include <linux/completion.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/dmaengine.h>
  11 #include <linux/irqreturn.h>
  12 #include <linux/jiffies.h>
  13 #include <linux/module.h>
  14 #include <linux/pci.h>
  15 #include <linux/platform_data/dma-dw.h>
  16 #include <linux/spi/spi.h>
  17 #include <linux/types.h>
  18
  19 #include "spi-dw.h"
  20
  21 #define DW_SPI_RX_BUSY          0
  22 #define DW_SPI_RX_BURST_LEVEL   16
  23 #define DW_SPI_TX_BUSY          1
  24 #define DW_SPI_TX_BURST_LEVEL   16
  25
  26 static bool dw_spi_dma_chan_filter(struct dma_chan *chan, void *param)
  27 {
  28         struct dw_dma_slave *s = param;
  29
  30         if (s->dma_dev != chan->device->dev)
  31                 return false;
  32
  33         chan->private = s;
  34         return true;
  35 }
  36
  37 static void dw_spi_dma_maxburst_init(struct dw_spi *dws)
  38 {
  39         struct dma_slave_caps caps;
  40         u32 max_burst, def_burst;
  41         int ret;
  42
  43         def_burst = dws->fifo_len / 2;
  44
  45         ret = dma_get_slave_caps(dws->rxchan, &caps);
  46         if (!ret && caps.max_burst)
  47                 max_burst = caps.max_burst;
  48         else
  49                 max_burst = DW_SPI_RX_BURST_LEVEL;
  50
  51         dws->rxburst = min(max_burst, def_burst);
  52         dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1);
  53
  54         ret = dma_get_slave_caps(dws->txchan, &caps);
  55         if (!ret && caps.max_burst)
  56                 max_burst = caps.max_burst;
  57         else
  58                 max_burst = DW_SPI_TX_BURST_LEVEL;
  59
  60         /*
  61          * Having a Rx DMA channel serviced with higher priority than a Tx DMA
  62          * channel might not be enough to provide a well balanced DMA-based
  63          * SPI transfer interface. There might still be moments when the Tx DMA
  64          * channel is occasionally handled faster than the Rx DMA channel.
  65          * That in its turn will eventually cause the SPI Rx FIFO overflow if
  66          * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's
  67          * cleared by the Rx DMA channel. In order to fix the problem the Tx
  68          * DMA activity is intentionally slowed down by limiting the SPI Tx
  69          * FIFO depth with a value twice bigger than the Tx burst length.
  70          */
  71         dws->txburst = min(max_burst, def_burst);
  72         dw_writel(dws, DW_SPI_DMATDLR, dws->txburst);
  73 }
  74
  75 static int dw_spi_dma_caps_init(struct dw_spi *dws)
  76 {
  77         struct dma_slave_caps tx, rx;
  78         int ret;
  79
  80         ret = dma_get_slave_caps(dws->txchan, &tx);
  81         if (ret)
  82                 return ret;
  83
  84         ret = dma_get_slave_caps(dws->rxchan, &rx);
  85         if (ret)
  86                 return ret;
  87
  88         if (!(tx.directions & BIT(DMA_MEM_TO_DEV) &&
  89               rx.directions & BIT(DMA_DEV_TO_MEM)))
  90                 return -ENXIO;
  91
  92         if (tx.max_sg_burst > 0 && rx.max_sg_burst > 0)
  93                 dws->dma_sg_burst = min(tx.max_sg_burst, rx.max_sg_burst);
  94         else if (tx.max_sg_burst > 0)
  95                 dws->dma_sg_burst = tx.max_sg_burst;
  96         else if (rx.max_sg_burst > 0)
  97                 dws->dma_sg_burst = rx.max_sg_burst;
  98         else
  99                 dws->dma_sg_burst = 0;
 100
 101         /*
 102          * Assuming both channels belong to the same DMA controller hence the
 103          * peripheral side address width capabilities most likely would be
 104          * the same.
 105          */
 106         dws->dma_addr_widths = tx.dst_addr_widths & rx.src_addr_widths;
 107
 108         return 0;
 109 }
 110
 111 static int dw_spi_dma_init_mfld(struct device *dev, struct dw_spi *dws)
 112 {
 113         struct dw_dma_slave dma_tx = { .dst_id = 1 }, *tx = &dma_tx;
 114         struct dw_dma_slave dma_rx = { .src_id = 0 }, *rx = &dma_rx;
 115         struct pci_dev *dma_dev;
 116         dma_cap_mask_t mask;
 117         int ret = -EBUSY;
 118
 119         /*
 120          * Get pci device for DMA controller, currently it could only
 121          * be the DMA controller of Medfield
 122          */
 123         dma_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x0827, NULL);
 124         if (!dma_dev)
 125                 return -ENODEV;
 126
 127         dma_cap_zero(mask);
 128         dma_cap_set(DMA_SLAVE, mask);
 129
 130         /* 1. Init rx channel */
 131         rx->dma_dev = &dma_dev->dev;
 132         dws->rxchan = dma_request_channel(mask, dw_spi_dma_chan_filter, rx);
 133         if (!dws->rxchan)
 134                 goto err_exit;
 135
 136         /* 2. Init tx channel */
 137         tx->dma_dev = &dma_dev->dev;
 138         dws->txchan = dma_request_channel(mask, dw_spi_dma_chan_filter, tx);
 139         if (!dws->txchan)
 140                 goto free_rxchan;
 141
 142         dws->host->dma_rx = dws->rxchan;
 143         dws->host->dma_tx = dws->txchan;
 144
 145         init_completion(&dws->dma_completion);
 146
 147         ret = dw_spi_dma_caps_init(dws);
 148         if (ret)
 149                 goto free_txchan;
 150
 151         dw_spi_dma_maxburst_init(dws);
 152
 153         pci_dev_put(dma_dev);
 154
 155         return 0;
 156
 157 free_txchan:
 158         dma_release_channel(dws->txchan);
 159         dws->txchan = NULL;
 160 free_rxchan:
 161         dma_release_channel(dws->rxchan);
 162         dws->rxchan = NULL;
 163 err_exit:
 164         pci_dev_put(dma_dev);
 165         return ret;
 166 }
 167
 168 static int dw_spi_dma_init_generic(struct device *dev, struct dw_spi *dws)
 169 {
 170         int ret;
 171
 172         dws->rxchan = dma_request_chan(dev, "rx");
 173         if (IS_ERR(dws->rxchan)) {
 174                 ret = PTR_ERR(dws->rxchan);
 175                 dws->rxchan = NULL;
 176                 goto err_exit;
 177         }
 178
 179         dws->txchan = dma_request_chan(dev, "tx");
 180         if (IS_ERR(dws->txchan)) {
 181                 ret = PTR_ERR(dws->txchan);
 182                 dws->txchan = NULL;
 183                 goto free_rxchan;
 184         }
 185
 186         dws->host->dma_rx = dws->rxchan;
 187         dws->host->dma_tx = dws->txchan;
 188
 189         init_completion(&dws->dma_completion);
 190
 191         ret = dw_spi_dma_caps_init(dws);
 192         if (ret)
 193                 goto free_txchan;
 194
 195         dw_spi_dma_maxburst_init(dws);
 196
 197         return 0;
 198
 199 free_txchan:
 200         dma_release_channel(dws->txchan);
 201         dws->txchan = NULL;
 202 free_rxchan:
 203         dma_release_channel(dws->rxchan);
 204         dws->rxchan = NULL;
 205 err_exit:
 206         return ret;
 207 }
 208
 209 static void dw_spi_dma_exit(struct dw_spi *dws)
 210 {
 211         if (dws->txchan) {
 212                 dmaengine_terminate_sync(dws->txchan);
 213                 dma_release_channel(dws->txchan);
 214         }
 215
 216         if (dws->rxchan) {
 217                 dmaengine_terminate_sync(dws->rxchan);
 218                 dma_release_channel(dws->rxchan);
 219         }
 220 }
 221
 222 static irqreturn_t dw_spi_dma_transfer_handler(struct dw_spi *dws)
 223 {
 224         dw_spi_check_status(dws, false);
 225
 226         complete(&dws->dma_completion);
 227
 228         return IRQ_HANDLED;
 229 }
 230
 231 static enum dma_slave_buswidth dw_spi_dma_convert_width(u8 n_bytes)
 232 {
 233         switch (n_bytes) {
 234         case 1:
 235                 return DMA_SLAVE_BUSWIDTH_1_BYTE;
 236         case 2:
 237                 return DMA_SLAVE_BUSWIDTH_2_BYTES;
 238         case 4:
 239                 return DMA_SLAVE_BUSWIDTH_4_BYTES;
 240         default:
 241                 return DMA_SLAVE_BUSWIDTH_UNDEFINED;
 242         }
 243 }
 244
 245 static bool dw_spi_can_dma(struct spi_controller *host,
 246                            struct spi_device *spi, struct spi_transfer *xfer)
 247 {
 248         struct dw_spi *dws = spi_controller_get_devdata(host);
 249         enum dma_slave_buswidth dma_bus_width;
 250
 251         if (xfer->len <= dws->fifo_len)
 252                 return false;
 253
 254         dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes);
 255
 256         return dws->dma_addr_widths & BIT(dma_bus_width);
 257 }
 258
 259 static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed)
 260 {
 261         unsigned long long ms;
 262
 263         ms = len * MSEC_PER_SEC * BITS_PER_BYTE;
 264         do_div(ms, speed);
 265         ms += ms + 200;
 266
 267         if (ms > UINT_MAX)
 268                 ms = UINT_MAX;
 269
 270         ms = wait_for_completion_timeout(&dws->dma_completion,
 271                                          msecs_to_jiffies(ms));
 272
 273         if (ms == 0) {
 274                 dev_err(&dws->host->cur_msg->spi->dev,
 275                         "DMA transaction timed out\n");
 276                 return -ETIMEDOUT;
 277         }
 278
 279         return 0;
 280 }
 281
 282 static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws)
 283 {
 284         return !(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_TF_EMPT);
 285 }
 286
 287 static int dw_spi_dma_wait_tx_done(struct dw_spi *dws,
 288                                    struct spi_transfer *xfer)
 289 {
 290         int retry = DW_SPI_WAIT_RETRIES;
 291         struct spi_delay delay;
 292         u32 nents;
 293
 294         nents = dw_readl(dws, DW_SPI_TXFLR);
 295         delay.unit = SPI_DELAY_UNIT_SCK;
 296         delay.value = nents * dws->n_bytes * BITS_PER_BYTE;
 297
 298         while (dw_spi_dma_tx_busy(dws) && retry--)
 299                 spi_delay_exec(&delay, xfer);
 300
 301         if (retry < 0) {
 302                 dev_err(&dws->host->dev, "Tx hanged up\n");
 303                 return -EIO;
 304         }
 305
 306         return 0;
 307 }
 308
 309 /*
 310  * dws->dma_chan_busy is set before the dma transfer starts, callback for tx
 311  * channel will clear a corresponding bit.
 312  */
 313 static void dw_spi_dma_tx_done(void *arg)
 314 {
 315         struct dw_spi *dws = arg;
 316
 317         clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy);
 318         if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) {
 319                 dw_writel(dws, DW_SPI_DMARDLR, 0);
 320                 return;
 321         }
 322
 323         complete(&dws->dma_completion);
 324 }
 325
 326 static int dw_spi_dma_config_tx(struct dw_spi *dws)
 327 {
 328         struct dma_slave_config txconf;
 329
 330         memset(&txconf, 0, sizeof(txconf));
 331         txconf.direction = DMA_MEM_TO_DEV;
 332         txconf.dst_addr = dws->dma_addr;
 333         txconf.dst_maxburst = dws->txburst;
 334         txconf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 335         txconf.dst_addr_width = dw_spi_dma_convert_width(dws->n_bytes);
 336         txconf.device_fc = false;
 337
 338         return dmaengine_slave_config(dws->txchan, &txconf);
 339 }
 340
 341 static int dw_spi_dma_submit_tx(struct dw_spi *dws, struct scatterlist *sgl,
 342                                 unsigned int nents)
 343 {
 344         struct dma_async_tx_descriptor *txdesc;
 345         dma_cookie_t cookie;
 346         int ret;
 347
 348         txdesc = dmaengine_prep_slave_sg(dws->txchan, sgl, nents,
 349                                          DMA_MEM_TO_DEV,
 350                                          DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 351         if (!txdesc)
 352                 return -ENOMEM;
 353
 354         txdesc->callback = dw_spi_dma_tx_done;
 355         txdesc->callback_param = dws;
 356
 357         cookie = dmaengine_submit(txdesc);
 358         ret = dma_submit_error(cookie);
 359         if (ret) {
 360                 dmaengine_terminate_sync(dws->txchan);
 361                 return ret;
 362         }
 363
 364         set_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy);
 365
 366         return 0;
 367 }
 368
 369 static inline bool dw_spi_dma_rx_busy(struct dw_spi *dws)
 370 {
 371         return !!(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_RF_NOT_EMPT);
 372 }
 373
 374 static int dw_spi_dma_wait_rx_done(struct dw_spi *dws)
 375 {
 376         int retry = DW_SPI_WAIT_RETRIES;
 377         struct spi_delay delay;
 378         unsigned long ns, us;
 379         u32 nents;
 380
 381         /*
 382          * It's unlikely that DMA engine is still doing the data fetching, but
 383          * if it's let's give it some reasonable time. The timeout calculation
 384          * is based on the synchronous APB/SSI reference clock rate, on a
 385          * number of data entries left in the Rx FIFO, times a number of clock
 386          * periods normally needed for a single APB read/write transaction
 387          * without PREADY signal utilized (which is true for the DW APB SSI
 388          * controller).
 389          */
 390         nents = dw_readl(dws, DW_SPI_RXFLR);
 391         ns = 4U * NSEC_PER_SEC / dws->max_freq * nents;
 392         if (ns <= NSEC_PER_USEC) {
 393                 delay.unit = SPI_DELAY_UNIT_NSECS;
 394                 delay.value = ns;
 395         } else {
 396                 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
 397                 delay.unit = SPI_DELAY_UNIT_USECS;
 398                 delay.value = clamp_val(us, 0, USHRT_MAX);
 399         }
 400
 401         while (dw_spi_dma_rx_busy(dws) && retry--)
 402                 spi_delay_exec(&delay, NULL);
 403
 404         if (retry < 0) {
 405                 dev_err(&dws->host->dev, "Rx hanged up\n");
 406                 return -EIO;
 407         }
 408
 409         return 0;
 410 }
 411
 412 /*
 413  * dws->dma_chan_busy is set before the dma transfer starts, callback for rx
 414  * channel will clear a corresponding bit.
 415  */
 416 static void dw_spi_dma_rx_done(void *arg)
 417 {
 418         struct dw_spi *dws = arg;
 419
 420         clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy);
 421         if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy))
 422                 return;
 423
 424         complete(&dws->dma_completion);
 425 }
 426
 427 static int dw_spi_dma_config_rx(struct dw_spi *dws)
 428 {
 429         struct dma_slave_config rxconf;
 430
 431         memset(&rxconf, 0, sizeof(rxconf));
 432         rxconf.direction = DMA_DEV_TO_MEM;
 433         rxconf.src_addr = dws->dma_addr;
 434         rxconf.src_maxburst = dws->rxburst;
 435         rxconf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 436         rxconf.src_addr_width = dw_spi_dma_convert_width(dws->n_bytes);
 437         rxconf.device_fc = false;
 438
 439         return dmaengine_slave_config(dws->rxchan, &rxconf);
 440 }
 441
 442 static int dw_spi_dma_submit_rx(struct dw_spi *dws, struct scatterlist *sgl,
 443                                 unsigned int nents)
 444 {
 445         struct dma_async_tx_descriptor *rxdesc;
 446         dma_cookie_t cookie;
 447         int ret;
 448
 449         rxdesc = dmaengine_prep_slave_sg(dws->rxchan, sgl, nents,
 450                                          DMA_DEV_TO_MEM,
 451                                          DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 452         if (!rxdesc)
 453                 return -ENOMEM;
 454
 455         rxdesc->callback = dw_spi_dma_rx_done;
 456         rxdesc->callback_param = dws;
 457
 458         cookie = dmaengine_submit(rxdesc);
 459         ret = dma_submit_error(cookie);
 460         if (ret) {
 461                 dmaengine_terminate_sync(dws->rxchan);
 462                 return ret;
 463         }
 464
 465         set_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy);
 466
 467         return 0;
 468 }
 469
 470 static int dw_spi_dma_setup(struct dw_spi *dws, struct spi_transfer *xfer)
 471 {
 472         u16 imr, dma_ctrl;
 473         int ret;
 474
 475         if (!xfer->tx_buf)
 476                 return -EINVAL;
 477
 478         /* Setup DMA channels */
 479         ret = dw_spi_dma_config_tx(dws);
 480         if (ret)
 481                 return ret;
 482
 483         if (xfer->rx_buf) {
 484                 ret = dw_spi_dma_config_rx(dws);
 485                 if (ret)
 486                         return ret;
 487         }
 488
 489         /* Set the DMA handshaking interface */
 490         dma_ctrl = DW_SPI_DMACR_TDMAE;
 491         if (xfer->rx_buf)
 492                 dma_ctrl |= DW_SPI_DMACR_RDMAE;
 493         dw_writel(dws, DW_SPI_DMACR, dma_ctrl);
 494
 495         /* Set the interrupt mask */
 496         imr = DW_SPI_INT_TXOI;
 497         if (xfer->rx_buf)
 498                 imr |= DW_SPI_INT_RXUI | DW_SPI_INT_RXOI;
 499         dw_spi_umask_intr(dws, imr);
 500
 501         reinit_completion(&dws->dma_completion);
 502
 503         dws->transfer_handler = dw_spi_dma_transfer_handler;
 504
 505         return 0;
 506 }
 507
 508 static int dw_spi_dma_transfer_all(struct dw_spi *dws,
 509                                    struct spi_transfer *xfer)
 510 {
 511         int ret;
 512
 513         /* Submit the DMA Tx transfer */
 514         ret = dw_spi_dma_submit_tx(dws, xfer->tx_sg.sgl, xfer->tx_sg.nents);
 515         if (ret)
 516                 goto err_clear_dmac;
 517
 518         /* Submit the DMA Rx transfer if required */
 519         if (xfer->rx_buf) {
 520                 ret = dw_spi_dma_submit_rx(dws, xfer->rx_sg.sgl,
 521                                            xfer->rx_sg.nents);
 522                 if (ret)
 523                         goto err_clear_dmac;
 524
 525                 /* rx must be started before tx due to spi instinct */
 526                 dma_async_issue_pending(dws->rxchan);
 527         }
 528
 529         dma_async_issue_pending(dws->txchan);
 530
 531         ret = dw_spi_dma_wait(dws, xfer->len, xfer->effective_speed_hz);
 532
 533 err_clear_dmac:
 534         dw_writel(dws, DW_SPI_DMACR, 0);
 535
 536         return ret;
 537 }
 538
 539 /*
 540  * In case if at least one of the requested DMA channels doesn't support the
 541  * hardware accelerated SG list entries traverse, the DMA driver will most
 542  * likely work that around by performing the IRQ-based SG list entries
 543  * resubmission. That might and will cause a problem if the DMA Tx channel is
 544  * recharged and re-executed before the Rx DMA channel. Due to
 545  * non-deterministic IRQ-handler execution latency the DMA Tx channel will
 546  * start pushing data to the SPI bus before the Rx DMA channel is even
 547  * reinitialized with the next inbound SG list entry. By doing so the DMA Tx
 548  * channel will implicitly start filling the DW APB SSI Rx FIFO up, which while
 549  * the DMA Rx channel being recharged and re-executed will eventually be
 550  * overflown.
 551  *
 552  * In order to solve the problem we have to feed the DMA engine with SG list
 553  * entries one-by-one. It shall keep the DW APB SSI Tx and Rx FIFOs
 554  * synchronized and prevent the Rx FIFO overflow. Since in general the tx_sg
 555  * and rx_sg lists may have different number of entries of different lengths
 556  * (though total length should match) let's virtually split the SG-lists to the
 557  * set of DMA transfers, which length is a minimum of the ordered SG-entries
 558  * lengths. An ASCII-sketch of the implemented algo is following:
 559  *                  xfer->len
 560  *                |___________|
 561  * tx_sg list:    |___|____|__|
 562  * rx_sg list:    |_|____|____|
 563  * DMA transfers: |_|_|__|_|__|
 564  *
 565  * Note in order to have this workaround solving the denoted problem the DMA
 566  * engine driver should properly initialize the max_sg_burst capability and set
 567  * the DMA device max segment size parameter with maximum data block size the
 568  * DMA engine supports.
 569  */
 570
 571 static int dw_spi_dma_transfer_one(struct dw_spi *dws,
 572                                    struct spi_transfer *xfer)
 573 {
 574         struct scatterlist *tx_sg = NULL, *rx_sg = NULL, tx_tmp, rx_tmp;
 575         unsigned int tx_len = 0, rx_len = 0;
 576         unsigned int base, len;
 577         int ret;
 578
 579         sg_init_table(&tx_tmp, 1);
 580         sg_init_table(&rx_tmp, 1);
 581
 582         for (base = 0, len = 0; base < xfer->len; base += len) {
 583                 /* Fetch next Tx DMA data chunk */
 584                 if (!tx_len) {
 585                         tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg);
 586                         sg_dma_address(&tx_tmp) = sg_dma_address(tx_sg);
 587                         tx_len = sg_dma_len(tx_sg);
 588                 }
 589
 590                 /* Fetch next Rx DMA data chunk */
 591                 if (!rx_len) {
 592                         rx_sg = !rx_sg ? &xfer->rx_sg.sgl[0] : sg_next(rx_sg);
 593                         sg_dma_address(&rx_tmp) = sg_dma_address(rx_sg);
 594                         rx_len = sg_dma_len(rx_sg);
 595                 }
 596
 597                 len = min(tx_len, rx_len);
 598
 599                 sg_dma_len(&tx_tmp) = len;
 600                 sg_dma_len(&rx_tmp) = len;
 601
 602                 /* Submit DMA Tx transfer */
 603                 ret = dw_spi_dma_submit_tx(dws, &tx_tmp, 1);
 604                 if (ret)
 605                         break;
 606
 607                 /* Submit DMA Rx transfer */
 608                 ret = dw_spi_dma_submit_rx(dws, &rx_tmp, 1);
 609                 if (ret)
 610                         break;
 611
 612                 /* Rx must be started before Tx due to SPI instinct */
 613                 dma_async_issue_pending(dws->rxchan);
 614
 615                 dma_async_issue_pending(dws->txchan);
 616
 617                 /*
 618                  * Here we only need to wait for the DMA transfer to be
 619                  * finished since SPI controller is kept enabled during the
 620                  * procedure this loop implements and there is no risk to lose
 621                  * data left in the Tx/Rx FIFOs.
 622                  */
 623                 ret = dw_spi_dma_wait(dws, len, xfer->effective_speed_hz);
 624                 if (ret)
 625                         break;
 626
 627                 reinit_completion(&dws->dma_completion);
 628
 629                 sg_dma_address(&tx_tmp) += len;
 630                 sg_dma_address(&rx_tmp) += len;
 631                 tx_len -= len;
 632                 rx_len -= len;
 633         }
 634
 635         dw_writel(dws, DW_SPI_DMACR, 0);
 636
 637         return ret;
 638 }
 639
 640 static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer)
 641 {
 642         unsigned int nents;
 643         int ret;
 644
 645         nents = max(xfer->tx_sg.nents, xfer->rx_sg.nents);
 646
 647         dw_writel(dws, DW_SPI_DMARDLR, xfer->tx_buf ? (dws->rxburst - 1) : 0);
 648
 649         /*
 650          * Execute normal DMA-based transfer (which submits the Rx and Tx SG
 651          * lists directly to the DMA engine at once) if either full hardware
 652          * accelerated SG list traverse is supported by both channels, or the
 653          * Tx-only SPI transfer is requested, or the DMA engine is capable to
 654          * handle both SG lists on hardware accelerated basis.
 655          */
 656         if (!dws->dma_sg_burst || !xfer->rx_buf || nents <= dws->dma_sg_burst)
 657                 ret = dw_spi_dma_transfer_all(dws, xfer);
 658         else
 659                 ret = dw_spi_dma_transfer_one(dws, xfer);
 660         if (ret)
 661                 return ret;
 662
 663         if (dws->host->cur_msg->status == -EINPROGRESS) {
 664                 ret = dw_spi_dma_wait_tx_done(dws, xfer);
 665                 if (ret)
 666                         return ret;
 667         }
 668
 669         if (xfer->rx_buf && dws->host->cur_msg->status == -EINPROGRESS)
 670                 ret = dw_spi_dma_wait_rx_done(dws);
 671
 672         return ret;
 673 }
 674
 675 static void dw_spi_dma_stop(struct dw_spi *dws)
 676 {
 677         if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) {
 678                 dmaengine_terminate_sync(dws->txchan);
 679                 clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy);
 680         }
 681         if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) {
 682                 dmaengine_terminate_sync(dws->rxchan);
 683                 clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy);
 684         }
 685 }
 686
 687 static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = {
 688         .dma_init       = dw_spi_dma_init_mfld,
 689         .dma_exit       = dw_spi_dma_exit,
 690         .dma_setup      = dw_spi_dma_setup,
 691         .can_dma        = dw_spi_can_dma,
 692         .dma_transfer   = dw_spi_dma_transfer,
 693         .dma_stop       = dw_spi_dma_stop,
 694 };
 695
 696 void dw_spi_dma_setup_mfld(struct dw_spi *dws)
 697 {
 698         dws->dma_ops = &dw_spi_dma_mfld_ops;
 699 }
 700 EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_mfld, SPI_DW_CORE);
 701
 702 static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = {
 703         .dma_init       = dw_spi_dma_init_generic,
 704         .dma_exit       = dw_spi_dma_exit,
 705         .dma_setup      = dw_spi_dma_setup,
 706         .can_dma        = dw_spi_can_dma,
 707         .dma_transfer   = dw_spi_dma_transfer,
 708         .dma_stop       = dw_spi_dma_stop,
 709 };
 710
 711 void dw_spi_dma_setup_generic(struct dw_spi *dws)
 712 {
 713         dws->dma_ops = &dw_spi_dma_generic_ops;
 714 }
 715 EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_generic, SPI_DW_CORE);