drivers/spi/spi-imx.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
   3 // Copyright (C) 2008 Juergen Beisert
   4
   5 #include <linux/clk.h>
   6 #include <linux/completion.h>
   7 #include <linux/delay.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/err.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/io.h>
  13 #include <linux/irq.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/pinctrl/consumer.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/pm_runtime.h>
  19 #include <linux/slab.h>
  20 #include <linux/spi/spi.h>
  21 #include <linux/types.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/property.h>
  25
  26 #include <linux/dma/imx-dma.h>
  27
  28 #define DRIVER_NAME "spi_imx"
  29
  30 static bool use_dma = true;
  31 module_param(use_dma, bool, 0644);
  32 MODULE_PARM_DESC(use_dma, "Enable usage of DMA when available (default)");
  33
  34 /* define polling limits */
  35 static unsigned int polling_limit_us = 30;
  36 module_param(polling_limit_us, uint, 0664);
  37 MODULE_PARM_DESC(polling_limit_us,
  38                  "time in us to run a transfer in polling mode\n");
  39
  40 #define MXC_RPM_TIMEOUT         2000 /* 2000ms */
  41
  42 #define MXC_CSPIRXDATA          0x00
  43 #define MXC_CSPITXDATA          0x04
  44 #define MXC_CSPICTRL            0x08
  45 #define MXC_CSPIINT             0x0c
  46 #define MXC_RESET               0x1c
  47
  48 /* generic defines to abstract from the different register layouts */
  49 #define MXC_INT_RR      (1 << 0) /* Receive data ready interrupt */
  50 #define MXC_INT_TE      (1 << 1) /* Transmit FIFO empty interrupt */
  51 #define MXC_INT_RDR     BIT(4) /* Receive date threshold interrupt */
  52
  53 /* The maximum bytes that a sdma BD can transfer. */
  54 #define MAX_SDMA_BD_BYTES (1 << 15)
  55 #define MX51_ECSPI_CTRL_MAX_BURST       512
  56 /* The maximum bytes that IMX53_ECSPI can transfer in slave mode.*/
  57 #define MX53_MAX_TRANSFER_BYTES         512
  58
  59 enum spi_imx_devtype {
  60         IMX1_CSPI,
  61         IMX21_CSPI,
  62         IMX27_CSPI,
  63         IMX31_CSPI,
  64         IMX35_CSPI,     /* CSPI on all i.mx except above */
  65         IMX51_ECSPI,    /* ECSPI on i.mx51 */
  66         IMX53_ECSPI,    /* ECSPI on i.mx53 and later */
  67 };
  68
  69 struct spi_imx_data;
  70
  71 struct spi_imx_devtype_data {
  72         void (*intctrl)(struct spi_imx_data *spi_imx, int enable);
  73         int (*prepare_message)(struct spi_imx_data *spi_imx, struct spi_message *msg);
  74         int (*prepare_transfer)(struct spi_imx_data *spi_imx, struct spi_device *spi);
  75         void (*trigger)(struct spi_imx_data *spi_imx);
  76         int (*rx_available)(struct spi_imx_data *spi_imx);
  77         void (*reset)(struct spi_imx_data *spi_imx);
  78         void (*setup_wml)(struct spi_imx_data *spi_imx);
  79         void (*disable)(struct spi_imx_data *spi_imx);
  80         void (*disable_dma)(struct spi_imx_data *spi_imx);
  81         bool has_dmamode;
  82         bool has_slavemode;
  83         unsigned int fifo_size;
  84         bool dynamic_burst;
  85         /*
  86          * ERR009165 fixed or not:
  87          * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
  88          */
  89         bool tx_glitch_fixed;
  90         enum spi_imx_devtype devtype;
  91 };
  92
  93 struct spi_imx_data {
  94         struct spi_controller *controller;
  95         struct device *dev;
  96
  97         struct completion xfer_done;
  98         void __iomem *base;
  99         unsigned long base_phys;
 100
 101         struct clk *clk_per;
 102         struct clk *clk_ipg;
 103         unsigned long spi_clk;
 104         unsigned int spi_bus_clk;
 105
 106         unsigned int bits_per_word;
 107         unsigned int spi_drctl;
 108
 109         unsigned int count, remainder;
 110         void (*tx)(struct spi_imx_data *spi_imx);
 111         void (*rx)(struct spi_imx_data *spi_imx);
 112         void *rx_buf;
 113         const void *tx_buf;
 114         unsigned int txfifo; /* number of words pushed in tx FIFO */
 115         unsigned int dynamic_burst;
 116         bool rx_only;
 117
 118         /* Slave mode */
 119         bool slave_mode;
 120         bool slave_aborted;
 121         unsigned int slave_burst;
 122
 123         /* DMA */
 124         bool usedma;
 125         u32 wml;
 126         struct completion dma_rx_completion;
 127         struct completion dma_tx_completion;
 128
 129         const struct spi_imx_devtype_data *devtype_data;
 130 };
 131
 132 static inline int is_imx27_cspi(struct spi_imx_data *d)
 133 {
 134         return d->devtype_data->devtype == IMX27_CSPI;
 135 }
 136
 137 static inline int is_imx35_cspi(struct spi_imx_data *d)
 138 {
 139         return d->devtype_data->devtype == IMX35_CSPI;
 140 }
 141
 142 static inline int is_imx51_ecspi(struct spi_imx_data *d)
 143 {
 144         return d->devtype_data->devtype == IMX51_ECSPI;
 145 }
 146
 147 static inline int is_imx53_ecspi(struct spi_imx_data *d)
 148 {
 149         return d->devtype_data->devtype == IMX53_ECSPI;
 150 }
 151
 152 #define MXC_SPI_BUF_RX(type)                                            \
 153 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)         \
 154 {                                                                       \
 155         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);       \
 156                                                                         \
 157         if (spi_imx->rx_buf) {                                          \
 158                 *(type *)spi_imx->rx_buf = val;                         \
 159                 spi_imx->rx_buf += sizeof(type);                        \
 160         }                                                               \
 161                                                                         \
 162         spi_imx->remainder -= sizeof(type);                             \
 163 }
 164
 165 #define MXC_SPI_BUF_TX(type)                                            \
 166 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)         \
 167 {                                                                       \
 168         type val = 0;                                                   \
 169                                                                         \
 170         if (spi_imx->tx_buf) {                                          \
 171                 val = *(type *)spi_imx->tx_buf;                         \
 172                 spi_imx->tx_buf += sizeof(type);                        \
 173         }                                                               \
 174                                                                         \
 175         spi_imx->count -= sizeof(type);                                 \
 176                                                                         \
 177         writel(val, spi_imx->base + MXC_CSPITXDATA);                    \
 178 }
 179
 180 MXC_SPI_BUF_RX(u8)
 181 MXC_SPI_BUF_TX(u8)
 182 MXC_SPI_BUF_RX(u16)
 183 MXC_SPI_BUF_TX(u16)
 184 MXC_SPI_BUF_RX(u32)
 185 MXC_SPI_BUF_TX(u32)
 186
 187 /* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
 188  * (which is currently not the case in this driver)
 189  */
 190 static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
 191         256, 384, 512, 768, 1024};
 192
 193 /* MX21, MX27 */
 194 static unsigned int spi_imx_clkdiv_1(unsigned int fin,
 195                 unsigned int fspi, unsigned int max, unsigned int *fres)
 196 {
 197         int i;
 198
 199         for (i = 2; i < max; i++)
 200                 if (fspi * mxc_clkdivs[i] >= fin)
 201                         break;
 202
 203         *fres = fin / mxc_clkdivs[i];
 204         return i;
 205 }
 206
 207 /* MX1, MX31, MX35, MX51 CSPI */
 208 static unsigned int spi_imx_clkdiv_2(unsigned int fin,
 209                 unsigned int fspi, unsigned int *fres)
 210 {
 211         int i, div = 4;
 212
 213         for (i = 0; i < 7; i++) {
 214                 if (fspi * div >= fin)
 215                         goto out;
 216                 div <<= 1;
 217         }
 218
 219 out:
 220         *fres = fin / div;
 221         return i;
 222 }
 223
 224 static int spi_imx_bytes_per_word(const int bits_per_word)
 225 {
 226         if (bits_per_word <= 8)
 227                 return 1;
 228         else if (bits_per_word <= 16)
 229                 return 2;
 230         else
 231                 return 4;
 232 }
 233
 234 static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device *spi,
 235                          struct spi_transfer *transfer)
 236 {
 237         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
 238
 239         if (!use_dma || controller->fallback)
 240                 return false;
 241
 242         if (!controller->dma_rx)
 243                 return false;
 244
 245         if (spi_imx->slave_mode)
 246                 return false;
 247
 248         if (transfer->len < spi_imx->devtype_data->fifo_size)
 249                 return false;
 250
 251         spi_imx->dynamic_burst = 0;
 252
 253         return true;
 254 }
 255
 256 #define MX51_ECSPI_CTRL         0x08
 257 #define MX51_ECSPI_CTRL_ENABLE          (1 <<  0)
 258 #define MX51_ECSPI_CTRL_XCH             (1 <<  2)
 259 #define MX51_ECSPI_CTRL_SMC             (1 << 3)
 260 #define MX51_ECSPI_CTRL_MODE_MASK       (0xf << 4)
 261 #define MX51_ECSPI_CTRL_DRCTL(drctl)    ((drctl) << 16)
 262 #define MX51_ECSPI_CTRL_POSTDIV_OFFSET  8
 263 #define MX51_ECSPI_CTRL_PREDIV_OFFSET   12
 264 #define MX51_ECSPI_CTRL_CS(cs)          ((cs) << 18)
 265 #define MX51_ECSPI_CTRL_BL_OFFSET       20
 266 #define MX51_ECSPI_CTRL_BL_MASK         (0xfff << 20)
 267
 268 #define MX51_ECSPI_CONFIG       0x0c
 269 #define MX51_ECSPI_CONFIG_SCLKPHA(cs)   (1 << ((cs) +  0))
 270 #define MX51_ECSPI_CONFIG_SCLKPOL(cs)   (1 << ((cs) +  4))
 271 #define MX51_ECSPI_CONFIG_SBBCTRL(cs)   (1 << ((cs) +  8))
 272 #define MX51_ECSPI_CONFIG_SSBPOL(cs)    (1 << ((cs) + 12))
 273 #define MX51_ECSPI_CONFIG_SCLKCTL(cs)   (1 << ((cs) + 20))
 274
 275 #define MX51_ECSPI_INT          0x10
 276 #define MX51_ECSPI_INT_TEEN             (1 <<  0)
 277 #define MX51_ECSPI_INT_RREN             (1 <<  3)
 278 #define MX51_ECSPI_INT_RDREN            (1 <<  4)
 279
 280 #define MX51_ECSPI_DMA          0x14
 281 #define MX51_ECSPI_DMA_TX_WML(wml)      ((wml) & 0x3f)
 282 #define MX51_ECSPI_DMA_RX_WML(wml)      (((wml) & 0x3f) << 16)
 283 #define MX51_ECSPI_DMA_RXT_WML(wml)     (((wml) & 0x3f) << 24)
 284
 285 #define MX51_ECSPI_DMA_TEDEN            (1 << 7)
 286 #define MX51_ECSPI_DMA_RXDEN            (1 << 23)
 287 #define MX51_ECSPI_DMA_RXTDEN           (1 << 31)
 288
 289 #define MX51_ECSPI_STAT         0x18
 290 #define MX51_ECSPI_STAT_RR              (1 <<  3)
 291
 292 #define MX51_ECSPI_TESTREG      0x20
 293 #define MX51_ECSPI_TESTREG_LBC  BIT(31)
 294
 295 static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
 296 {
 297         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);
 298
 299         if (spi_imx->rx_buf) {
 300 #ifdef __LITTLE_ENDIAN
 301                 unsigned int bytes_per_word;
 302
 303                 bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 304                 if (bytes_per_word == 1)
 305                         swab32s(&val);
 306                 else if (bytes_per_word == 2)
 307                         swahw32s(&val);
 308 #endif
 309                 *(u32 *)spi_imx->rx_buf = val;
 310                 spi_imx->rx_buf += sizeof(u32);
 311         }
 312
 313         spi_imx->remainder -= sizeof(u32);
 314 }
 315
 316 static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
 317 {
 318         int unaligned;
 319         u32 val;
 320
 321         unaligned = spi_imx->remainder % 4;
 322
 323         if (!unaligned) {
 324                 spi_imx_buf_rx_swap_u32(spi_imx);
 325                 return;
 326         }
 327
 328         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 329                 spi_imx_buf_rx_u16(spi_imx);
 330                 return;
 331         }
 332
 333         val = readl(spi_imx->base + MXC_CSPIRXDATA);
 334
 335         while (unaligned--) {
 336                 if (spi_imx->rx_buf) {
 337                         *(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;
 338                         spi_imx->rx_buf++;
 339                 }
 340                 spi_imx->remainder--;
 341         }
 342 }
 343
 344 static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
 345 {
 346         u32 val = 0;
 347 #ifdef __LITTLE_ENDIAN
 348         unsigned int bytes_per_word;
 349 #endif
 350
 351         if (spi_imx->tx_buf) {
 352                 val = *(u32 *)spi_imx->tx_buf;
 353                 spi_imx->tx_buf += sizeof(u32);
 354         }
 355
 356         spi_imx->count -= sizeof(u32);
 357 #ifdef __LITTLE_ENDIAN
 358         bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 359
 360         if (bytes_per_word == 1)
 361                 swab32s(&val);
 362         else if (bytes_per_word == 2)
 363                 swahw32s(&val);
 364 #endif
 365         writel(val, spi_imx->base + MXC_CSPITXDATA);
 366 }
 367
 368 static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
 369 {
 370         int unaligned;
 371         u32 val = 0;
 372
 373         unaligned = spi_imx->count % 4;
 374
 375         if (!unaligned) {
 376                 spi_imx_buf_tx_swap_u32(spi_imx);
 377                 return;
 378         }
 379
 380         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 381                 spi_imx_buf_tx_u16(spi_imx);
 382                 return;
 383         }
 384
 385         while (unaligned--) {
 386                 if (spi_imx->tx_buf) {
 387                         val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);
 388                         spi_imx->tx_buf++;
 389                 }
 390                 spi_imx->count--;
 391         }
 392
 393         writel(val, spi_imx->base + MXC_CSPITXDATA);
 394 }
 395
 396 static void mx53_ecspi_rx_slave(struct spi_imx_data *spi_imx)
 397 {
 398         u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));
 399
 400         if (spi_imx->rx_buf) {
 401                 int n_bytes = spi_imx->slave_burst % sizeof(val);
 402
 403                 if (!n_bytes)
 404                         n_bytes = sizeof(val);
 405
 406                 memcpy(spi_imx->rx_buf,
 407                        ((u8 *)&val) + sizeof(val) - n_bytes, n_bytes);
 408
 409                 spi_imx->rx_buf += n_bytes;
 410                 spi_imx->slave_burst -= n_bytes;
 411         }
 412
 413         spi_imx->remainder -= sizeof(u32);
 414 }
 415
 416 static void mx53_ecspi_tx_slave(struct spi_imx_data *spi_imx)
 417 {
 418         u32 val = 0;
 419         int n_bytes = spi_imx->count % sizeof(val);
 420
 421         if (!n_bytes)
 422                 n_bytes = sizeof(val);
 423
 424         if (spi_imx->tx_buf) {
 425                 memcpy(((u8 *)&val) + sizeof(val) - n_bytes,
 426                        spi_imx->tx_buf, n_bytes);
 427                 val = cpu_to_be32(val);
 428                 spi_imx->tx_buf += n_bytes;
 429         }
 430
 431         spi_imx->count -= n_bytes;
 432
 433         writel(val, spi_imx->base + MXC_CSPITXDATA);
 434 }
 435
 436 /* MX51 eCSPI */
 437 static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
 438                                       unsigned int fspi, unsigned int *fres)
 439 {
 440         /*
 441          * there are two 4-bit dividers, the pre-divider divides by
 442          * $pre, the post-divider by 2^$post
 443          */
 444         unsigned int pre, post;
 445         unsigned int fin = spi_imx->spi_clk;
 446
 447         fspi = min(fspi, fin);
 448
 449         post = fls(fin) - fls(fspi);
 450         if (fin > fspi << post)
 451                 post++;
 452
 453         /* now we have: (fin <= fspi << post) with post being minimal */
 454
 455         post = max(4U, post) - 4;
 456         if (unlikely(post > 0xf)) {
 457                 dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",
 458                                 fspi, fin);
 459                 return 0xff;
 460         }
 461
 462         pre = DIV_ROUND_UP(fin, fspi << post) - 1;
 463
 464         dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",
 465                         __func__, fin, fspi, post, pre);
 466
 467         /* Resulting frequency for the SCLK line. */
 468         *fres = (fin / (pre + 1)) >> post;
 469
 470         return (pre << MX51_ECSPI_CTRL_PREDIV_OFFSET) |
 471                 (post << MX51_ECSPI_CTRL_POSTDIV_OFFSET);
 472 }
 473
 474 static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)
 475 {
 476         unsigned int val = 0;
 477
 478         if (enable & MXC_INT_TE)
 479                 val |= MX51_ECSPI_INT_TEEN;
 480
 481         if (enable & MXC_INT_RR)
 482                 val |= MX51_ECSPI_INT_RREN;
 483
 484         if (enable & MXC_INT_RDR)
 485                 val |= MX51_ECSPI_INT_RDREN;
 486
 487         writel(val, spi_imx->base + MX51_ECSPI_INT);
 488 }
 489
 490 static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)
 491 {
 492         u32 reg;
 493
 494         reg = readl(spi_imx->base + MX51_ECSPI_CTRL);
 495         reg |= MX51_ECSPI_CTRL_XCH;
 496         writel(reg, spi_imx->base + MX51_ECSPI_CTRL);
 497 }
 498
 499 static void mx51_disable_dma(struct spi_imx_data *spi_imx)
 500 {
 501         writel(0, spi_imx->base + MX51_ECSPI_DMA);
 502 }
 503
 504 static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)
 505 {
 506         u32 ctrl;
 507
 508         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 509         ctrl &= ~MX51_ECSPI_CTRL_ENABLE;
 510         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 511 }
 512
 513 static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 514                                       struct spi_message *msg)
 515 {
 516         struct spi_device *spi = msg->spi;
 517         struct spi_transfer *xfer;
 518         u32 ctrl = MX51_ECSPI_CTRL_ENABLE;
 519         u32 min_speed_hz = ~0U;
 520         u32 testreg, delay;
 521         u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 522         u32 current_cfg = cfg;
 523
 524         /* set Master or Slave mode */
 525         if (spi_imx->slave_mode)
 526                 ctrl &= ~MX51_ECSPI_CTRL_MODE_MASK;
 527         else
 528                 ctrl |= MX51_ECSPI_CTRL_MODE_MASK;
 529
 530         /*
 531          * Enable SPI_RDY handling (falling edge/level triggered).
 532          */
 533         if (spi->mode & SPI_READY)
 534                 ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
 535
 536         /* set chip select to use */
 537         ctrl |= MX51_ECSPI_CTRL_CS(spi->chip_select);
 538
 539         /*
 540          * The ctrl register must be written first, with the EN bit set other
 541          * registers must not be written to.
 542          */
 543         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 544
 545         testreg = readl(spi_imx->base + MX51_ECSPI_TESTREG);
 546         if (spi->mode & SPI_LOOP)
 547                 testreg |= MX51_ECSPI_TESTREG_LBC;
 548         else
 549                 testreg &= ~MX51_ECSPI_TESTREG_LBC;
 550         writel(testreg, spi_imx->base + MX51_ECSPI_TESTREG);
 551
 552         /*
 553          * eCSPI burst completion by Chip Select signal in Slave mode
 554          * is not functional for imx53 Soc, config SPI burst completed when
 555          * BURST_LENGTH + 1 bits are received
 556          */
 557         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 558                 cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
 559         else
 560                 cfg |= MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
 561
 562         if (spi->mode & SPI_CPOL) {
 563                 cfg |= MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
 564                 cfg |= MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
 565         } else {
 566                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
 567                 cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
 568         }
 569
 570         if (spi->mode & SPI_CS_HIGH)
 571                 cfg |= MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
 572         else
 573                 cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
 574
 575         if (cfg == current_cfg)
 576                 return 0;
 577
 578         writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 579
 580         /*
 581          * Wait until the changes in the configuration register CONFIGREG
 582          * propagate into the hardware. It takes exactly one tick of the
 583          * SCLK clock, but we will wait two SCLK clock just to be sure. The
 584          * effect of the delay it takes for the hardware to apply changes
 585          * is noticable if the SCLK clock run very slow. In such a case, if
 586          * the polarity of SCLK should be inverted, the GPIO ChipSelect might
 587          * be asserted before the SCLK polarity changes, which would disrupt
 588          * the SPI communication as the device on the other end would consider
 589          * the change of SCLK polarity as a clock tick already.
 590          *
 591          * Because spi_imx->spi_bus_clk is only set in prepare_message
 592          * callback, iterate over all the transfers in spi_message, find the
 593          * one with lowest bus frequency, and use that bus frequency for the
 594          * delay calculation. In case all transfers have speed_hz == 0, then
 595          * min_speed_hz is ~0 and the resulting delay is zero.
 596          */
 597         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 598                 if (!xfer->speed_hz)
 599                         continue;
 600                 min_speed_hz = min(xfer->speed_hz, min_speed_hz);
 601         }
 602
 603         delay = (2 * 1000000) / min_speed_hz;
 604         if (likely(delay < 10)) /* SCLK is faster than 200 kHz */
 605                 udelay(delay);
 606         else                    /* SCLK is _very_ slow */
 607                 usleep_range(delay, delay + 10);
 608
 609         return 0;
 610 }
 611
 612 static void mx51_configure_cpha(struct spi_imx_data *spi_imx,
 613                                 struct spi_device *spi)
 614 {
 615         bool cpha = (spi->mode & SPI_CPHA);
 616         bool flip_cpha = (spi->mode & SPI_RX_CPHA_FLIP) && spi_imx->rx_only;
 617         u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 618
 619         /* Flip cpha logical value iff flip_cpha */
 620         cpha ^= flip_cpha;
 621
 622         if (cpha)
 623                 cfg |= MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
 624         else
 625                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
 626
 627         writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 628 }
 629
 630 static int mx51_ecspi_prepare_transfer(struct spi_imx_data *spi_imx,
 631                                        struct spi_device *spi)
 632 {
 633         u32 ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 634         u32 clk;
 635
 636         /* Clear BL field and set the right value */
 637         ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
 638         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 639                 ctrl |= (spi_imx->slave_burst * 8 - 1)
 640                         << MX51_ECSPI_CTRL_BL_OFFSET;
 641         else
 642                 ctrl |= (spi_imx->bits_per_word - 1)
 643                         << MX51_ECSPI_CTRL_BL_OFFSET;
 644
 645         /* set clock speed */
 646         ctrl &= ~(0xf << MX51_ECSPI_CTRL_POSTDIV_OFFSET |
 647                   0xf << MX51_ECSPI_CTRL_PREDIV_OFFSET);
 648         ctrl |= mx51_ecspi_clkdiv(spi_imx, spi_imx->spi_bus_clk, &clk);
 649         spi_imx->spi_bus_clk = clk;
 650
 651         mx51_configure_cpha(spi_imx, spi);
 652
 653         /*
 654          * ERR009165: work in XHC mode instead of SMC as PIO on the chips
 655          * before i.mx6ul.
 656          */
 657         if (spi_imx->usedma && spi_imx->devtype_data->tx_glitch_fixed)
 658                 ctrl |= MX51_ECSPI_CTRL_SMC;
 659         else
 660                 ctrl &= ~MX51_ECSPI_CTRL_SMC;
 661
 662         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 663
 664         return 0;
 665 }
 666
 667 static void mx51_setup_wml(struct spi_imx_data *spi_imx)
 668 {
 669         u32 tx_wml = 0;
 670
 671         if (spi_imx->devtype_data->tx_glitch_fixed)
 672                 tx_wml = spi_imx->wml;
 673         /*
 674          * Configure the DMA register: setup the watermark
 675          * and enable DMA request.
 676          */
 677         writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) |
 678                 MX51_ECSPI_DMA_TX_WML(tx_wml) |
 679                 MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |
 680                 MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN |
 681                 MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA);
 682 }
 683
 684 static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)
 685 {
 686         return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;
 687 }
 688
 689 static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 690 {
 691         /* drain receive buffer */
 692         while (mx51_ecspi_rx_available(spi_imx))
 693                 readl(spi_imx->base + MXC_CSPIRXDATA);
 694 }
 695
 696 #define MX31_INTREG_TEEN        (1 << 0)
 697 #define MX31_INTREG_RREN        (1 << 3)
 698
 699 #define MX31_CSPICTRL_ENABLE    (1 << 0)
 700 #define MX31_CSPICTRL_MASTER    (1 << 1)
 701 #define MX31_CSPICTRL_XCH       (1 << 2)
 702 #define MX31_CSPICTRL_SMC       (1 << 3)
 703 #define MX31_CSPICTRL_POL       (1 << 4)
 704 #define MX31_CSPICTRL_PHA       (1 << 5)
 705 #define MX31_CSPICTRL_SSCTL     (1 << 6)
 706 #define MX31_CSPICTRL_SSPOL     (1 << 7)
 707 #define MX31_CSPICTRL_BC_SHIFT  8
 708 #define MX35_CSPICTRL_BL_SHIFT  20
 709 #define MX31_CSPICTRL_CS_SHIFT  24
 710 #define MX35_CSPICTRL_CS_SHIFT  12
 711 #define MX31_CSPICTRL_DR_SHIFT  16
 712
 713 #define MX31_CSPI_DMAREG        0x10
 714 #define MX31_DMAREG_RH_DEN      (1<<4)
 715 #define MX31_DMAREG_TH_DEN      (1<<1)
 716
 717 #define MX31_CSPISTATUS         0x14
 718 #define MX31_STATUS_RR          (1 << 3)
 719
 720 #define MX31_CSPI_TESTREG       0x1C
 721 #define MX31_TEST_LBC           (1 << 14)
 722
 723 /* These functions also work for the i.MX35, but be aware that
 724  * the i.MX35 has a slightly different register layout for bits
 725  * we do not use here.
 726  */
 727 static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)
 728 {
 729         unsigned int val = 0;
 730
 731         if (enable & MXC_INT_TE)
 732                 val |= MX31_INTREG_TEEN;
 733         if (enable & MXC_INT_RR)
 734                 val |= MX31_INTREG_RREN;
 735
 736         writel(val, spi_imx->base + MXC_CSPIINT);
 737 }
 738
 739 static void mx31_trigger(struct spi_imx_data *spi_imx)
 740 {
 741         unsigned int reg;
 742
 743         reg = readl(spi_imx->base + MXC_CSPICTRL);
 744         reg |= MX31_CSPICTRL_XCH;
 745         writel(reg, spi_imx->base + MXC_CSPICTRL);
 746 }
 747
 748 static int mx31_prepare_message(struct spi_imx_data *spi_imx,
 749                                 struct spi_message *msg)
 750 {
 751         return 0;
 752 }
 753
 754 static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,
 755                                  struct spi_device *spi)
 756 {
 757         unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_MASTER;
 758         unsigned int clk;
 759
 760         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 761                 MX31_CSPICTRL_DR_SHIFT;
 762         spi_imx->spi_bus_clk = clk;
 763
 764         if (is_imx35_cspi(spi_imx)) {
 765                 reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;
 766                 reg |= MX31_CSPICTRL_SSCTL;
 767         } else {
 768                 reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;
 769         }
 770
 771         if (spi->mode & SPI_CPHA)
 772                 reg |= MX31_CSPICTRL_PHA;
 773         if (spi->mode & SPI_CPOL)
 774                 reg |= MX31_CSPICTRL_POL;
 775         if (spi->mode & SPI_CS_HIGH)
 776                 reg |= MX31_CSPICTRL_SSPOL;
 777         if (!spi->cs_gpiod)
 778                 reg |= (spi->chip_select) <<
 779                         (is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 780                                                   MX31_CSPICTRL_CS_SHIFT);
 781
 782         if (spi_imx->usedma)
 783                 reg |= MX31_CSPICTRL_SMC;
 784
 785         writel(reg, spi_imx->base + MXC_CSPICTRL);
 786
 787         reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
 788         if (spi->mode & SPI_LOOP)
 789                 reg |= MX31_TEST_LBC;
 790         else
 791                 reg &= ~MX31_TEST_LBC;
 792         writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
 793
 794         if (spi_imx->usedma) {
 795                 /*
 796                  * configure DMA requests when RXFIFO is half full and
 797                  * when TXFIFO is half empty
 798                  */
 799                 writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
 800                         spi_imx->base + MX31_CSPI_DMAREG);
 801         }
 802
 803         return 0;
 804 }
 805
 806 static int mx31_rx_available(struct spi_imx_data *spi_imx)
 807 {
 808         return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;
 809 }
 810
 811 static void mx31_reset(struct spi_imx_data *spi_imx)
 812 {
 813         /* drain receive buffer */
 814         while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)
 815                 readl(spi_imx->base + MXC_CSPIRXDATA);
 816 }
 817
 818 #define MX21_INTREG_RR          (1 << 4)
 819 #define MX21_INTREG_TEEN        (1 << 9)
 820 #define MX21_INTREG_RREN        (1 << 13)
 821
 822 #define MX21_CSPICTRL_POL       (1 << 5)
 823 #define MX21_CSPICTRL_PHA       (1 << 6)
 824 #define MX21_CSPICTRL_SSPOL     (1 << 8)
 825 #define MX21_CSPICTRL_XCH       (1 << 9)
 826 #define MX21_CSPICTRL_ENABLE    (1 << 10)
 827 #define MX21_CSPICTRL_MASTER    (1 << 11)
 828 #define MX21_CSPICTRL_DR_SHIFT  14
 829 #define MX21_CSPICTRL_CS_SHIFT  19
 830
 831 static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)
 832 {
 833         unsigned int val = 0;
 834
 835         if (enable & MXC_INT_TE)
 836                 val |= MX21_INTREG_TEEN;
 837         if (enable & MXC_INT_RR)
 838                 val |= MX21_INTREG_RREN;
 839
 840         writel(val, spi_imx->base + MXC_CSPIINT);
 841 }
 842
 843 static void mx21_trigger(struct spi_imx_data *spi_imx)
 844 {
 845         unsigned int reg;
 846
 847         reg = readl(spi_imx->base + MXC_CSPICTRL);
 848         reg |= MX21_CSPICTRL_XCH;
 849         writel(reg, spi_imx->base + MXC_CSPICTRL);
 850 }
 851
 852 static int mx21_prepare_message(struct spi_imx_data *spi_imx,
 853                                 struct spi_message *msg)
 854 {
 855         return 0;
 856 }
 857
 858 static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,
 859                                  struct spi_device *spi)
 860 {
 861         unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_MASTER;
 862         unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;
 863         unsigned int clk;
 864
 865         reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, spi_imx->spi_bus_clk, max, &clk)
 866                 << MX21_CSPICTRL_DR_SHIFT;
 867         spi_imx->spi_bus_clk = clk;
 868
 869         reg |= spi_imx->bits_per_word - 1;
 870
 871         if (spi->mode & SPI_CPHA)
 872                 reg |= MX21_CSPICTRL_PHA;
 873         if (spi->mode & SPI_CPOL)
 874                 reg |= MX21_CSPICTRL_POL;
 875         if (spi->mode & SPI_CS_HIGH)
 876                 reg |= MX21_CSPICTRL_SSPOL;
 877         if (!spi->cs_gpiod)
 878                 reg |= spi->chip_select << MX21_CSPICTRL_CS_SHIFT;
 879
 880         writel(reg, spi_imx->base + MXC_CSPICTRL);
 881
 882         return 0;
 883 }
 884
 885 static int mx21_rx_available(struct spi_imx_data *spi_imx)
 886 {
 887         return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;
 888 }
 889
 890 static void mx21_reset(struct spi_imx_data *spi_imx)
 891 {
 892         writel(1, spi_imx->base + MXC_RESET);
 893 }
 894
 895 #define MX1_INTREG_RR           (1 << 3)
 896 #define MX1_INTREG_TEEN         (1 << 8)
 897 #define MX1_INTREG_RREN         (1 << 11)
 898
 899 #define MX1_CSPICTRL_POL        (1 << 4)
 900 #define MX1_CSPICTRL_PHA        (1 << 5)
 901 #define MX1_CSPICTRL_XCH        (1 << 8)
 902 #define MX1_CSPICTRL_ENABLE     (1 << 9)
 903 #define MX1_CSPICTRL_MASTER     (1 << 10)
 904 #define MX1_CSPICTRL_DR_SHIFT   13
 905
 906 static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)
 907 {
 908         unsigned int val = 0;
 909
 910         if (enable & MXC_INT_TE)
 911                 val |= MX1_INTREG_TEEN;
 912         if (enable & MXC_INT_RR)
 913                 val |= MX1_INTREG_RREN;
 914
 915         writel(val, spi_imx->base + MXC_CSPIINT);
 916 }
 917
 918 static void mx1_trigger(struct spi_imx_data *spi_imx)
 919 {
 920         unsigned int reg;
 921
 922         reg = readl(spi_imx->base + MXC_CSPICTRL);
 923         reg |= MX1_CSPICTRL_XCH;
 924         writel(reg, spi_imx->base + MXC_CSPICTRL);
 925 }
 926
 927 static int mx1_prepare_message(struct spi_imx_data *spi_imx,
 928                                struct spi_message *msg)
 929 {
 930         return 0;
 931 }
 932
 933 static int mx1_prepare_transfer(struct spi_imx_data *spi_imx,
 934                                 struct spi_device *spi)
 935 {
 936         unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_MASTER;
 937         unsigned int clk;
 938
 939         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 940                 MX1_CSPICTRL_DR_SHIFT;
 941         spi_imx->spi_bus_clk = clk;
 942
 943         reg |= spi_imx->bits_per_word - 1;
 944
 945         if (spi->mode & SPI_CPHA)
 946                 reg |= MX1_CSPICTRL_PHA;
 947         if (spi->mode & SPI_CPOL)
 948                 reg |= MX1_CSPICTRL_POL;
 949
 950         writel(reg, spi_imx->base + MXC_CSPICTRL);
 951
 952         return 0;
 953 }
 954
 955 static int mx1_rx_available(struct spi_imx_data *spi_imx)
 956 {
 957         return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;
 958 }
 959
 960 static void mx1_reset(struct spi_imx_data *spi_imx)
 961 {
 962         writel(1, spi_imx->base + MXC_RESET);
 963 }
 964
 965 static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
 966         .intctrl = mx1_intctrl,
 967         .prepare_message = mx1_prepare_message,
 968         .prepare_transfer = mx1_prepare_transfer,
 969         .trigger = mx1_trigger,
 970         .rx_available = mx1_rx_available,
 971         .reset = mx1_reset,
 972         .fifo_size = 8,
 973         .has_dmamode = false,
 974         .dynamic_burst = false,
 975         .has_slavemode = false,
 976         .devtype = IMX1_CSPI,
 977 };
 978
 979 static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
 980         .intctrl = mx21_intctrl,
 981         .prepare_message = mx21_prepare_message,
 982         .prepare_transfer = mx21_prepare_transfer,
 983         .trigger = mx21_trigger,
 984         .rx_available = mx21_rx_available,
 985         .reset = mx21_reset,
 986         .fifo_size = 8,
 987         .has_dmamode = false,
 988         .dynamic_burst = false,
 989         .has_slavemode = false,
 990         .devtype = IMX21_CSPI,
 991 };
 992
 993 static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
 994         /* i.mx27 cspi shares the functions with i.mx21 one */
 995         .intctrl = mx21_intctrl,
 996         .prepare_message = mx21_prepare_message,
 997         .prepare_transfer = mx21_prepare_transfer,
 998         .trigger = mx21_trigger,
 999         .rx_available = mx21_rx_available,
1000         .reset = mx21_reset,
1001         .fifo_size = 8,
1002         .has_dmamode = false,
1003         .dynamic_burst = false,
1004         .has_slavemode = false,
1005         .devtype = IMX27_CSPI,
1006 };
1007
1008 static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
1009         .intctrl = mx31_intctrl,
1010         .prepare_message = mx31_prepare_message,
1011         .prepare_transfer = mx31_prepare_transfer,
1012         .trigger = mx31_trigger,
1013         .rx_available = mx31_rx_available,
1014         .reset = mx31_reset,
1015         .fifo_size = 8,
1016         .has_dmamode = false,
1017         .dynamic_burst = false,
1018         .has_slavemode = false,
1019         .devtype = IMX31_CSPI,
1020 };
1021
1022 static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
1023         /* i.mx35 and later cspi shares the functions with i.mx31 one */
1024         .intctrl = mx31_intctrl,
1025         .prepare_message = mx31_prepare_message,
1026         .prepare_transfer = mx31_prepare_transfer,
1027         .trigger = mx31_trigger,
1028         .rx_available = mx31_rx_available,
1029         .reset = mx31_reset,
1030         .fifo_size = 8,
1031         .has_dmamode = true,
1032         .dynamic_burst = false,
1033         .has_slavemode = false,
1034         .devtype = IMX35_CSPI,
1035 };
1036
1037 static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
1038         .intctrl = mx51_ecspi_intctrl,
1039         .prepare_message = mx51_ecspi_prepare_message,
1040         .prepare_transfer = mx51_ecspi_prepare_transfer,
1041         .trigger = mx51_ecspi_trigger,
1042         .rx_available = mx51_ecspi_rx_available,
1043         .reset = mx51_ecspi_reset,
1044         .setup_wml = mx51_setup_wml,
1045         .disable_dma = mx51_disable_dma,
1046         .fifo_size = 64,
1047         .has_dmamode = true,
1048         .dynamic_burst = true,
1049         .has_slavemode = true,
1050         .disable = mx51_ecspi_disable,
1051         .devtype = IMX51_ECSPI,
1052 };
1053
1054 static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
1055         .intctrl = mx51_ecspi_intctrl,
1056         .prepare_message = mx51_ecspi_prepare_message,
1057         .prepare_transfer = mx51_ecspi_prepare_transfer,
1058         .trigger = mx51_ecspi_trigger,
1059         .rx_available = mx51_ecspi_rx_available,
1060         .disable_dma = mx51_disable_dma,
1061         .reset = mx51_ecspi_reset,
1062         .fifo_size = 64,
1063         .has_dmamode = true,
1064         .has_slavemode = true,
1065         .disable = mx51_ecspi_disable,
1066         .devtype = IMX53_ECSPI,
1067 };
1068
1069 static struct spi_imx_devtype_data imx6ul_ecspi_devtype_data = {
1070         .intctrl = mx51_ecspi_intctrl,
1071         .prepare_message = mx51_ecspi_prepare_message,
1072         .prepare_transfer = mx51_ecspi_prepare_transfer,
1073         .trigger = mx51_ecspi_trigger,
1074         .rx_available = mx51_ecspi_rx_available,
1075         .reset = mx51_ecspi_reset,
1076         .setup_wml = mx51_setup_wml,
1077         .fifo_size = 64,
1078         .has_dmamode = true,
1079         .dynamic_burst = true,
1080         .has_slavemode = true,
1081         .tx_glitch_fixed = true,
1082         .disable = mx51_ecspi_disable,
1083         .devtype = IMX51_ECSPI,
1084 };
1085
1086 static const struct of_device_id spi_imx_dt_ids[] = {
1087         { .compatible = "fsl,imx1-cspi", .data = &imx1_cspi_devtype_data, },
1088         { .compatible = "fsl,imx21-cspi", .data = &imx21_cspi_devtype_data, },
1089         { .compatible = "fsl,imx27-cspi", .data = &imx27_cspi_devtype_data, },
1090         { .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
1091         { .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
1092         { .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
1093         { .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
1094         { .compatible = "fsl,imx6ul-ecspi", .data = &imx6ul_ecspi_devtype_data, },
1095         { /* sentinel */ }
1096 };
1097 MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
1098
1099 static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)
1100 {
1101         u32 ctrl;
1102
1103         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
1104         ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
1105         ctrl |= ((n_bits - 1) << MX51_ECSPI_CTRL_BL_OFFSET);
1106         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
1107 }
1108
1109 static void spi_imx_push(struct spi_imx_data *spi_imx)
1110 {
1111         unsigned int burst_len;
1112
1113         /*
1114          * Reload the FIFO when the remaining bytes to be transferred in the
1115          * current burst is 0. This only applies when bits_per_word is a
1116          * multiple of 8.
1117          */
1118         if (!spi_imx->remainder) {
1119                 if (spi_imx->dynamic_burst) {
1120
1121                         /* We need to deal unaligned data first */
1122                         burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;
1123
1124                         if (!burst_len)
1125                                 burst_len = MX51_ECSPI_CTRL_MAX_BURST;
1126
1127                         spi_imx_set_burst_len(spi_imx, burst_len * 8);
1128
1129                         spi_imx->remainder = burst_len;
1130                 } else {
1131                         spi_imx->remainder = spi_imx_bytes_per_word(spi_imx->bits_per_word);
1132                 }
1133         }
1134
1135         while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
1136                 if (!spi_imx->count)
1137                         break;
1138                 if (spi_imx->dynamic_burst &&
1139                     spi_imx->txfifo >= DIV_ROUND_UP(spi_imx->remainder, 4))
1140                         break;
1141                 spi_imx->tx(spi_imx);
1142                 spi_imx->txfifo++;
1143         }
1144
1145         if (!spi_imx->slave_mode)
1146                 spi_imx->devtype_data->trigger(spi_imx);
1147 }
1148
1149 static irqreturn_t spi_imx_isr(int irq, void *dev_id)
1150 {
1151         struct spi_imx_data *spi_imx = dev_id;
1152
1153         while (spi_imx->txfifo &&
1154                spi_imx->devtype_data->rx_available(spi_imx)) {
1155                 spi_imx->rx(spi_imx);
1156                 spi_imx->txfifo--;
1157         }
1158
1159         if (spi_imx->count) {
1160                 spi_imx_push(spi_imx);
1161                 return IRQ_HANDLED;
1162         }
1163
1164         if (spi_imx->txfifo) {
1165                 /* No data left to push, but still waiting for rx data,
1166                  * enable receive data available interrupt.
1167                  */
1168                 spi_imx->devtype_data->intctrl(
1169                                 spi_imx, MXC_INT_RR);
1170                 return IRQ_HANDLED;
1171         }
1172
1173         spi_imx->devtype_data->intctrl(spi_imx, 0);
1174         complete(&spi_imx->xfer_done);
1175
1176         return IRQ_HANDLED;
1177 }
1178
1179 static int spi_imx_dma_configure(struct spi_controller *controller)
1180 {
1181         int ret;
1182         enum dma_slave_buswidth buswidth;
1183         struct dma_slave_config rx = {}, tx = {};
1184         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1185
1186         switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
1187         case 4:
1188                 buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
1189                 break;
1190         case 2:
1191                 buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
1192                 break;
1193         case 1:
1194                 buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
1195                 break;
1196         default:
1197                 return -EINVAL;
1198         }
1199
1200         tx.direction = DMA_MEM_TO_DEV;
1201         tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
1202         tx.dst_addr_width = buswidth;
1203         tx.dst_maxburst = spi_imx->wml;
1204         ret = dmaengine_slave_config(controller->dma_tx, &tx);
1205         if (ret) {
1206                 dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
1207                 return ret;
1208         }
1209
1210         rx.direction = DMA_DEV_TO_MEM;
1211         rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
1212         rx.src_addr_width = buswidth;
1213         rx.src_maxburst = spi_imx->wml;
1214         ret = dmaengine_slave_config(controller->dma_rx, &rx);
1215         if (ret) {
1216                 dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
1217                 return ret;
1218         }
1219
1220         return 0;
1221 }
1222
1223 static int spi_imx_setupxfer(struct spi_device *spi,
1224                                  struct spi_transfer *t)
1225 {
1226         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1227
1228         if (!t)
1229                 return 0;
1230
1231         if (!t->speed_hz) {
1232                 if (!spi->max_speed_hz) {
1233                         dev_err(&spi->dev, "no speed_hz provided!\n");
1234                         return -EINVAL;
1235                 }
1236                 dev_dbg(&spi->dev, "using spi->max_speed_hz!\n");
1237                 spi_imx->spi_bus_clk = spi->max_speed_hz;
1238         } else
1239                 spi_imx->spi_bus_clk = t->speed_hz;
1240
1241         spi_imx->bits_per_word = t->bits_per_word;
1242
1243         /*
1244          * Initialize the functions for transfer. To transfer non byte-aligned
1245          * words, we have to use multiple word-size bursts, we can't use
1246          * dynamic_burst in that case.
1247          */
1248         if (spi_imx->devtype_data->dynamic_burst && !spi_imx->slave_mode &&
1249             !(spi->mode & SPI_CS_WORD) &&
1250             (spi_imx->bits_per_word == 8 ||
1251             spi_imx->bits_per_word == 16 ||
1252             spi_imx->bits_per_word == 32)) {
1253
1254                 spi_imx->rx = spi_imx_buf_rx_swap;
1255                 spi_imx->tx = spi_imx_buf_tx_swap;
1256                 spi_imx->dynamic_burst = 1;
1257
1258         } else {
1259                 if (spi_imx->bits_per_word <= 8) {
1260                         spi_imx->rx = spi_imx_buf_rx_u8;
1261                         spi_imx->tx = spi_imx_buf_tx_u8;
1262                 } else if (spi_imx->bits_per_word <= 16) {
1263                         spi_imx->rx = spi_imx_buf_rx_u16;
1264                         spi_imx->tx = spi_imx_buf_tx_u16;
1265                 } else {
1266                         spi_imx->rx = spi_imx_buf_rx_u32;
1267                         spi_imx->tx = spi_imx_buf_tx_u32;
1268                 }
1269                 spi_imx->dynamic_burst = 0;
1270         }
1271
1272         if (spi_imx_can_dma(spi_imx->controller, spi, t))
1273                 spi_imx->usedma = true;
1274         else
1275                 spi_imx->usedma = false;
1276
1277         spi_imx->rx_only = ((t->tx_buf == NULL)
1278                         || (t->tx_buf == spi->controller->dummy_tx));
1279
1280         if (is_imx53_ecspi(spi_imx) && spi_imx->slave_mode) {
1281                 spi_imx->rx = mx53_ecspi_rx_slave;
1282                 spi_imx->tx = mx53_ecspi_tx_slave;
1283                 spi_imx->slave_burst = t->len;
1284         }
1285
1286         spi_imx->devtype_data->prepare_transfer(spi_imx, spi);
1287
1288         return 0;
1289 }
1290
1291 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
1292 {
1293         struct spi_controller *controller = spi_imx->controller;
1294
1295         if (controller->dma_rx) {
1296                 dma_release_channel(controller->dma_rx);
1297                 controller->dma_rx = NULL;
1298         }
1299
1300         if (controller->dma_tx) {
1301                 dma_release_channel(controller->dma_tx);
1302                 controller->dma_tx = NULL;
1303         }
1304 }
1305
1306 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
1307                              struct spi_controller *controller)
1308 {
1309         int ret;
1310
1311         spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
1312
1313         /* Prepare for TX DMA: */
1314         controller->dma_tx = dma_request_chan(dev, "tx");
1315         if (IS_ERR(controller->dma_tx)) {
1316                 ret = PTR_ERR(controller->dma_tx);
1317                 dev_dbg(dev, "can't get the TX DMA channel, error %d!\n", ret);
1318                 controller->dma_tx = NULL;
1319                 goto err;
1320         }
1321
1322         /* Prepare for RX : */
1323         controller->dma_rx = dma_request_chan(dev, "rx");
1324         if (IS_ERR(controller->dma_rx)) {
1325                 ret = PTR_ERR(controller->dma_rx);
1326                 dev_dbg(dev, "can't get the RX DMA channel, error %d\n", ret);
1327                 controller->dma_rx = NULL;
1328                 goto err;
1329         }
1330
1331         init_completion(&spi_imx->dma_rx_completion);
1332         init_completion(&spi_imx->dma_tx_completion);
1333         controller->can_dma = spi_imx_can_dma;
1334         controller->max_dma_len = MAX_SDMA_BD_BYTES;
1335         spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
1336                                          SPI_CONTROLLER_MUST_TX;
1337
1338         return 0;
1339 err:
1340         spi_imx_sdma_exit(spi_imx);
1341         return ret;
1342 }
1343
1344 static void spi_imx_dma_rx_callback(void *cookie)
1345 {
1346         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1347
1348         complete(&spi_imx->dma_rx_completion);
1349 }
1350
1351 static void spi_imx_dma_tx_callback(void *cookie)
1352 {
1353         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1354
1355         complete(&spi_imx->dma_tx_completion);
1356 }
1357
1358 static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
1359 {
1360         unsigned long timeout = 0;
1361
1362         /* Time with actual data transfer and CS change delay related to HW */
1363         timeout = (8 + 4) * size / spi_imx->spi_bus_clk;
1364
1365         /* Add extra second for scheduler related activities */
1366         timeout += 1;
1367
1368         /* Double calculated timeout */
1369         return msecs_to_jiffies(2 * timeout * MSEC_PER_SEC);
1370 }
1371
1372 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
1373                                 struct spi_transfer *transfer)
1374 {
1375         struct dma_async_tx_descriptor *desc_tx, *desc_rx;
1376         unsigned long transfer_timeout;
1377         unsigned long timeout;
1378         struct spi_controller *controller = spi_imx->controller;
1379         struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
1380         struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);
1381         unsigned int bytes_per_word, i;
1382         int ret;
1383
1384         /* Get the right burst length from the last sg to ensure no tail data */
1385         bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
1386         for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
1387                 if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
1388                         break;
1389         }
1390         /* Use 1 as wml in case no available burst length got */
1391         if (i == 0)
1392                 i = 1;
1393
1394         spi_imx->wml =  i;
1395
1396         ret = spi_imx_dma_configure(controller);
1397         if (ret)
1398                 goto dma_failure_no_start;
1399
1400         if (!spi_imx->devtype_data->setup_wml) {
1401                 dev_err(spi_imx->dev, "No setup_wml()?\n");
1402                 ret = -EINVAL;
1403                 goto dma_failure_no_start;
1404         }
1405         spi_imx->devtype_data->setup_wml(spi_imx);
1406
1407         /*
1408          * The TX DMA setup starts the transfer, so make sure RX is configured
1409          * before TX.
1410          */
1411         desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,
1412                                 rx->sgl, rx->nents, DMA_DEV_TO_MEM,
1413                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1414         if (!desc_rx) {
1415                 ret = -EINVAL;
1416                 goto dma_failure_no_start;
1417         }
1418
1419         desc_rx->callback = spi_imx_dma_rx_callback;
1420         desc_rx->callback_param = (void *)spi_imx;
1421         dmaengine_submit(desc_rx);
1422         reinit_completion(&spi_imx->dma_rx_completion);
1423         dma_async_issue_pending(controller->dma_rx);
1424
1425         desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,
1426                                 tx->sgl, tx->nents, DMA_MEM_TO_DEV,
1427                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1428         if (!desc_tx) {
1429                 dmaengine_terminate_all(controller->dma_tx);
1430                 dmaengine_terminate_all(controller->dma_rx);
1431                 return -EINVAL;
1432         }
1433
1434         desc_tx->callback = spi_imx_dma_tx_callback;
1435         desc_tx->callback_param = (void *)spi_imx;
1436         dmaengine_submit(desc_tx);
1437         reinit_completion(&spi_imx->dma_tx_completion);
1438         dma_async_issue_pending(controller->dma_tx);
1439
1440         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1441
1442         /* Wait SDMA to finish the data transfer.*/
1443         timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
1444                                                 transfer_timeout);
1445         if (!timeout) {
1446                 dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
1447                 dmaengine_terminate_all(controller->dma_tx);
1448                 dmaengine_terminate_all(controller->dma_rx);
1449                 return -ETIMEDOUT;
1450         }
1451
1452         timeout = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
1453                                               transfer_timeout);
1454         if (!timeout) {
1455                 dev_err(&controller->dev, "I/O Error in DMA RX\n");
1456                 spi_imx->devtype_data->reset(spi_imx);
1457                 dmaengine_terminate_all(controller->dma_rx);
1458                 return -ETIMEDOUT;
1459         }
1460
1461         return 0;
1462 /* fallback to pio */
1463 dma_failure_no_start:
1464         transfer->error |= SPI_TRANS_FAIL_NO_START;
1465         return ret;
1466 }
1467
1468 static int spi_imx_pio_transfer(struct spi_device *spi,
1469                                 struct spi_transfer *transfer)
1470 {
1471         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1472         unsigned long transfer_timeout;
1473         unsigned long timeout;
1474
1475         spi_imx->tx_buf = transfer->tx_buf;
1476         spi_imx->rx_buf = transfer->rx_buf;
1477         spi_imx->count = transfer->len;
1478         spi_imx->txfifo = 0;
1479         spi_imx->remainder = 0;
1480
1481         reinit_completion(&spi_imx->xfer_done);
1482
1483         spi_imx_push(spi_imx);
1484
1485         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);
1486
1487         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1488
1489         timeout = wait_for_completion_timeout(&spi_imx->xfer_done,
1490                                               transfer_timeout);
1491         if (!timeout) {
1492                 dev_err(&spi->dev, "I/O Error in PIO\n");
1493                 spi_imx->devtype_data->reset(spi_imx);
1494                 return -ETIMEDOUT;
1495         }
1496
1497         return 0;
1498 }
1499
1500 static int spi_imx_poll_transfer(struct spi_device *spi,
1501                                  struct spi_transfer *transfer)
1502 {
1503         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1504         unsigned long timeout;
1505
1506         spi_imx->tx_buf = transfer->tx_buf;
1507         spi_imx->rx_buf = transfer->rx_buf;
1508         spi_imx->count = transfer->len;
1509         spi_imx->txfifo = 0;
1510         spi_imx->remainder = 0;
1511
1512         /* fill in the fifo before timeout calculations if we are
1513          * interrupted here, then the data is getting transferred by
1514          * the HW while we are interrupted
1515          */
1516         spi_imx_push(spi_imx);
1517
1518         timeout = spi_imx_calculate_timeout(spi_imx, transfer->len) + jiffies;
1519         while (spi_imx->txfifo) {
1520                 /* RX */
1521                 while (spi_imx->txfifo &&
1522                        spi_imx->devtype_data->rx_available(spi_imx)) {
1523                         spi_imx->rx(spi_imx);
1524                         spi_imx->txfifo--;
1525                 }
1526
1527                 /* TX */
1528                 if (spi_imx->count) {
1529                         spi_imx_push(spi_imx);
1530                         continue;
1531                 }
1532
1533                 if (spi_imx->txfifo &&
1534                     time_after(jiffies, timeout)) {
1535
1536                         dev_err_ratelimited(&spi->dev,
1537                                             "timeout period reached: jiffies: %lu- falling back to interrupt mode\n",
1538                                             jiffies - timeout);
1539
1540                         /* fall back to interrupt mode */
1541                         return spi_imx_pio_transfer(spi, transfer);
1542                 }
1543         }
1544
1545         return 0;
1546 }
1547
1548 static int spi_imx_pio_transfer_slave(struct spi_device *spi,
1549                                       struct spi_transfer *transfer)
1550 {
1551         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1552         int ret = 0;
1553
1554         if (is_imx53_ecspi(spi_imx) &&
1555             transfer->len > MX53_MAX_TRANSFER_BYTES) {
1556                 dev_err(&spi->dev, "Transaction too big, max size is %d bytes\n",
1557                         MX53_MAX_TRANSFER_BYTES);
1558                 return -EMSGSIZE;
1559         }
1560
1561         spi_imx->tx_buf = transfer->tx_buf;
1562         spi_imx->rx_buf = transfer->rx_buf;
1563         spi_imx->count = transfer->len;
1564         spi_imx->txfifo = 0;
1565         spi_imx->remainder = 0;
1566
1567         reinit_completion(&spi_imx->xfer_done);
1568         spi_imx->slave_aborted = false;
1569
1570         spi_imx_push(spi_imx);
1571
1572         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);
1573
1574         if (wait_for_completion_interruptible(&spi_imx->xfer_done) ||
1575             spi_imx->slave_aborted) {
1576                 dev_dbg(&spi->dev, "interrupted\n");
1577                 ret = -EINTR;
1578         }
1579
1580         /* ecspi has a HW issue when works in Slave mode,
1581          * after 64 words writtern to TXFIFO, even TXFIFO becomes empty,
1582          * ECSPI_TXDATA keeps shift out the last word data,
1583          * so we have to disable ECSPI when in slave mode after the
1584          * transfer completes
1585          */
1586         if (spi_imx->devtype_data->disable)
1587                 spi_imx->devtype_data->disable(spi_imx);
1588
1589         return ret;
1590 }
1591
1592 static int spi_imx_transfer_one(struct spi_controller *controller,
1593                                 struct spi_device *spi,
1594                                 struct spi_transfer *transfer)
1595 {
1596         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1597         unsigned long hz_per_byte, byte_limit;
1598
1599         spi_imx_setupxfer(spi, transfer);
1600         transfer->effective_speed_hz = spi_imx->spi_bus_clk;
1601
1602         /* flush rxfifo before transfer */
1603         while (spi_imx->devtype_data->rx_available(spi_imx))
1604                 readl(spi_imx->base + MXC_CSPIRXDATA);
1605
1606         if (spi_imx->slave_mode)
1607                 return spi_imx_pio_transfer_slave(spi, transfer);
1608
1609         /*
1610          * If we decided in spi_imx_can_dma() that we want to do a DMA
1611          * transfer, the SPI transfer has already been mapped, so we
1612          * have to do the DMA transfer here.
1613          */
1614         if (spi_imx->usedma)
1615                 return spi_imx_dma_transfer(spi_imx, transfer);
1616         /*
1617          * Calculate the estimated time in us the transfer runs. Find
1618          * the number of Hz per byte per polling limit.
1619          */
1620         hz_per_byte = polling_limit_us ? ((8 + 4) * USEC_PER_SEC) / polling_limit_us : 0;
1621         byte_limit = hz_per_byte ? transfer->effective_speed_hz / hz_per_byte : 1;
1622
1623         /* run in polling mode for short transfers */
1624         if (transfer->len < byte_limit)
1625                 return spi_imx_poll_transfer(spi, transfer);
1626
1627         return spi_imx_pio_transfer(spi, transfer);
1628 }
1629
1630 static int spi_imx_setup(struct spi_device *spi)
1631 {
1632         dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
1633                  spi->mode, spi->bits_per_word, spi->max_speed_hz);
1634
1635         return 0;
1636 }
1637
1638 static void spi_imx_cleanup(struct spi_device *spi)
1639 {
1640 }
1641
1642 static int
1643 spi_imx_prepare_message(struct spi_controller *controller, struct spi_message *msg)
1644 {
1645         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1646         int ret;
1647
1648         ret = pm_runtime_resume_and_get(spi_imx->dev);
1649         if (ret < 0) {
1650                 dev_err(spi_imx->dev, "failed to enable clock\n");
1651                 return ret;
1652         }
1653
1654         ret = spi_imx->devtype_data->prepare_message(spi_imx, msg);
1655         if (ret) {
1656                 pm_runtime_mark_last_busy(spi_imx->dev);
1657                 pm_runtime_put_autosuspend(spi_imx->dev);
1658         }
1659
1660         return ret;
1661 }
1662
1663 static int
1664 spi_imx_unprepare_message(struct spi_controller *controller, struct spi_message *msg)
1665 {
1666         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1667
1668         pm_runtime_mark_last_busy(spi_imx->dev);
1669         pm_runtime_put_autosuspend(spi_imx->dev);
1670         return 0;
1671 }
1672
1673 static int spi_imx_slave_abort(struct spi_controller *controller)
1674 {
1675         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1676
1677         spi_imx->slave_aborted = true;
1678         complete(&spi_imx->xfer_done);
1679
1680         return 0;
1681 }
1682
1683 static int spi_imx_probe(struct platform_device *pdev)
1684 {
1685         struct device_node *np = pdev->dev.of_node;
1686         struct spi_controller *controller;
1687         struct spi_imx_data *spi_imx;
1688         struct resource *res;
1689         int ret, irq, spi_drctl;
1690         const struct spi_imx_devtype_data *devtype_data =
1691                         of_device_get_match_data(&pdev->dev);
1692         bool slave_mode;
1693         u32 val;
1694
1695         slave_mode = devtype_data->has_slavemode &&
1696                         of_property_read_bool(np, "spi-slave");
1697         if (slave_mode)
1698                 controller = spi_alloc_slave(&pdev->dev,
1699                                              sizeof(struct spi_imx_data));
1700         else
1701                 controller = spi_alloc_master(&pdev->dev,
1702                                               sizeof(struct spi_imx_data));
1703         if (!controller)
1704                 return -ENOMEM;
1705
1706         ret = of_property_read_u32(np, "fsl,spi-rdy-drctl", &spi_drctl);
1707         if ((ret < 0) || (spi_drctl >= 0x3)) {
1708                 /* '11' is reserved */
1709                 spi_drctl = 0;
1710         }
1711
1712         platform_set_drvdata(pdev, controller);
1713
1714         controller->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
1715         controller->bus_num = np ? -1 : pdev->id;
1716         controller->use_gpio_descriptors = true;
1717
1718         spi_imx = spi_controller_get_devdata(controller);
1719         spi_imx->controller = controller;
1720         spi_imx->dev = &pdev->dev;
1721         spi_imx->slave_mode = slave_mode;
1722
1723         spi_imx->devtype_data = devtype_data;
1724
1725         /*
1726          * Get number of chip selects from device properties. This can be
1727          * coming from device tree or boardfiles, if it is not defined,
1728          * a default value of 3 chip selects will be used, as all the legacy
1729          * board files have <= 3 chip selects.
1730          */
1731         if (!device_property_read_u32(&pdev->dev, "num-cs", &val))
1732                 controller->num_chipselect = val;
1733         else
1734                 controller->num_chipselect = 3;
1735
1736         spi_imx->controller->transfer_one = spi_imx_transfer_one;
1737         spi_imx->controller->setup = spi_imx_setup;
1738         spi_imx->controller->cleanup = spi_imx_cleanup;
1739         spi_imx->controller->prepare_message = spi_imx_prepare_message;
1740         spi_imx->controller->unprepare_message = spi_imx_unprepare_message;
1741         spi_imx->controller->slave_abort = spi_imx_slave_abort;
1742         spi_imx->controller->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_NO_CS;
1743
1744         if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||
1745             is_imx53_ecspi(spi_imx))
1746                 spi_imx->controller->mode_bits |= SPI_LOOP | SPI_READY;
1747
1748         if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx))
1749                 spi_imx->controller->mode_bits |= SPI_RX_CPHA_FLIP;
1750
1751         if (is_imx51_ecspi(spi_imx) &&
1752             device_property_read_u32(&pdev->dev, "cs-gpios", NULL))
1753                 /*
1754                  * When using HW-CS implementing SPI_CS_WORD can be done by just
1755                  * setting the burst length to the word size. This is
1756                  * considerably faster than manually controlling the CS.
1757                  */
1758                 spi_imx->controller->mode_bits |= SPI_CS_WORD;
1759
1760         spi_imx->spi_drctl = spi_drctl;
1761
1762         init_completion(&spi_imx->xfer_done);
1763
1764         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1765         spi_imx->base = devm_ioremap_resource(&pdev->dev, res);
1766         if (IS_ERR(spi_imx->base)) {
1767                 ret = PTR_ERR(spi_imx->base);
1768                 goto out_controller_put;
1769         }
1770         spi_imx->base_phys = res->start;
1771
1772         irq = platform_get_irq(pdev, 0);
1773         if (irq < 0) {
1774                 ret = irq;
1775                 goto out_controller_put;
1776         }
1777
1778         ret = devm_request_irq(&pdev->dev, irq, spi_imx_isr, 0,
1779                                dev_name(&pdev->dev), spi_imx);
1780         if (ret) {
1781                 dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
1782                 goto out_controller_put;
1783         }
1784
1785         spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1786         if (IS_ERR(spi_imx->clk_ipg)) {
1787                 ret = PTR_ERR(spi_imx->clk_ipg);
1788                 goto out_controller_put;
1789         }
1790
1791         spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");
1792         if (IS_ERR(spi_imx->clk_per)) {
1793                 ret = PTR_ERR(spi_imx->clk_per);
1794                 goto out_controller_put;
1795         }
1796
1797         ret = clk_prepare_enable(spi_imx->clk_per);
1798         if (ret)
1799                 goto out_controller_put;
1800
1801         ret = clk_prepare_enable(spi_imx->clk_ipg);
1802         if (ret)
1803                 goto out_put_per;
1804
1805         pm_runtime_set_autosuspend_delay(spi_imx->dev, MXC_RPM_TIMEOUT);
1806         pm_runtime_use_autosuspend(spi_imx->dev);
1807         pm_runtime_get_noresume(spi_imx->dev);
1808         pm_runtime_set_active(spi_imx->dev);
1809         pm_runtime_enable(spi_imx->dev);
1810
1811         spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);
1812         /*
1813          * Only validated on i.mx35 and i.mx6 now, can remove the constraint
1814          * if validated on other chips.
1815          */
1816         if (spi_imx->devtype_data->has_dmamode) {
1817                 ret = spi_imx_sdma_init(&pdev->dev, spi_imx, controller);
1818                 if (ret == -EPROBE_DEFER)
1819                         goto out_runtime_pm_put;
1820
1821                 if (ret < 0)
1822                         dev_dbg(&pdev->dev, "dma setup error %d, use pio\n",
1823                                 ret);
1824         }
1825
1826         spi_imx->devtype_data->reset(spi_imx);
1827
1828         spi_imx->devtype_data->intctrl(spi_imx, 0);
1829
1830         controller->dev.of_node = pdev->dev.of_node;
1831         ret = spi_register_controller(controller);
1832         if (ret) {
1833                 dev_err_probe(&pdev->dev, ret, "register controller failed\n");
1834                 goto out_register_controller;
1835         }
1836
1837         pm_runtime_mark_last_busy(spi_imx->dev);
1838         pm_runtime_put_autosuspend(spi_imx->dev);
1839
1840         return ret;
1841
1842 out_register_controller:
1843         if (spi_imx->devtype_data->has_dmamode)
1844                 spi_imx_sdma_exit(spi_imx);
1845 out_runtime_pm_put:
1846         pm_runtime_dont_use_autosuspend(spi_imx->dev);
1847         pm_runtime_set_suspended(&pdev->dev);
1848         pm_runtime_disable(spi_imx->dev);
1849
1850         clk_disable_unprepare(spi_imx->clk_ipg);
1851 out_put_per:
1852         clk_disable_unprepare(spi_imx->clk_per);
1853 out_controller_put:
1854         spi_controller_put(controller);
1855
1856         return ret;
1857 }
1858
1859 static int spi_imx_remove(struct platform_device *pdev)
1860 {
1861         struct spi_controller *controller = platform_get_drvdata(pdev);
1862         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1863         int ret;
1864
1865         spi_unregister_controller(controller);
1866
1867         ret = pm_runtime_resume_and_get(spi_imx->dev);
1868         if (ret < 0) {
1869                 dev_err(spi_imx->dev, "failed to enable clock\n");
1870                 return ret;
1871         }
1872
1873         writel(0, spi_imx->base + MXC_CSPICTRL);
1874
1875         pm_runtime_dont_use_autosuspend(spi_imx->dev);
1876         pm_runtime_put_sync(spi_imx->dev);
1877         pm_runtime_disable(spi_imx->dev);
1878
1879         spi_imx_sdma_exit(spi_imx);
1880
1881         return 0;
1882 }
1883
1884 static int __maybe_unused spi_imx_runtime_resume(struct device *dev)
1885 {
1886         struct spi_controller *controller = dev_get_drvdata(dev);
1887         struct spi_imx_data *spi_imx;
1888         int ret;
1889
1890         spi_imx = spi_controller_get_devdata(controller);
1891
1892         ret = clk_prepare_enable(spi_imx->clk_per);
1893         if (ret)
1894                 return ret;
1895
1896         ret = clk_prepare_enable(spi_imx->clk_ipg);
1897         if (ret) {
1898                 clk_disable_unprepare(spi_imx->clk_per);
1899                 return ret;
1900         }
1901
1902         return 0;
1903 }
1904
1905 static int __maybe_unused spi_imx_runtime_suspend(struct device *dev)
1906 {
1907         struct spi_controller *controller = dev_get_drvdata(dev);
1908         struct spi_imx_data *spi_imx;
1909
1910         spi_imx = spi_controller_get_devdata(controller);
1911
1912         clk_disable_unprepare(spi_imx->clk_per);
1913         clk_disable_unprepare(spi_imx->clk_ipg);
1914
1915         return 0;
1916 }
1917
1918 static int __maybe_unused spi_imx_suspend(struct device *dev)
1919 {
1920         pinctrl_pm_select_sleep_state(dev);
1921         return 0;
1922 }
1923
1924 static int __maybe_unused spi_imx_resume(struct device *dev)
1925 {
1926         pinctrl_pm_select_default_state(dev);
1927         return 0;
1928 }
1929
1930 static const struct dev_pm_ops imx_spi_pm = {
1931         SET_RUNTIME_PM_OPS(spi_imx_runtime_suspend,
1932                                 spi_imx_runtime_resume, NULL)
1933         SET_SYSTEM_SLEEP_PM_OPS(spi_imx_suspend, spi_imx_resume)
1934 };
1935
1936 static struct platform_driver spi_imx_driver = {
1937         .driver = {
1938                    .name = DRIVER_NAME,
1939                    .of_match_table = spi_imx_dt_ids,
1940                    .pm = &imx_spi_pm,
1941         },
1942         .probe = spi_imx_probe,
1943         .remove = spi_imx_remove,
1944 };
1945 module_platform_driver(spi_imx_driver);
1946
1947 MODULE_DESCRIPTION("i.MX SPI Controller driver");
1948 MODULE_AUTHOR("Sascha Hauer, Pengutronix");
1949 MODULE_LICENSE("GPL");
1950 MODULE_ALIAS("platform:" DRIVER_NAME);