drivers/spi/spi-omap2-mcspi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * OMAP2 McSPI controller driver
   4  *
   5  * Copyright (C) 2005, 2006 Nokia Corporation
   6  * Author:      Samuel Ortiz <samuel.ortiz@nokia.com> and
   7  *              Juha Yrjola <juha.yrjola@nokia.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/device.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/err.h>
  20 #include <linux/clk.h>
  21 #include <linux/io.h>
  22 #include <linux/slab.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/of.h>
  25 #include <linux/of_device.h>
  26 #include <linux/gcd.h>
  27
  28 #include <linux/spi/spi.h>
  29
  30 #include <linux/platform_data/spi-omap2-mcspi.h>
  31
  32 #define OMAP2_MCSPI_MAX_FREQ            48000000
  33 #define OMAP2_MCSPI_MAX_DIVIDER         4096
  34 #define OMAP2_MCSPI_MAX_FIFODEPTH       64
  35 #define OMAP2_MCSPI_MAX_FIFOWCNT        0xFFFF
  36 #define SPI_AUTOSUSPEND_TIMEOUT         2000
  37
  38 #define OMAP2_MCSPI_REVISION            0x00
  39 #define OMAP2_MCSPI_SYSSTATUS           0x14
  40 #define OMAP2_MCSPI_IRQSTATUS           0x18
  41 #define OMAP2_MCSPI_IRQENABLE           0x1c
  42 #define OMAP2_MCSPI_WAKEUPENABLE        0x20
  43 #define OMAP2_MCSPI_SYST                0x24
  44 #define OMAP2_MCSPI_MODULCTRL           0x28
  45 #define OMAP2_MCSPI_XFERLEVEL           0x7c
  46
  47 /* per-channel banks, 0x14 bytes each, first is: */
  48 #define OMAP2_MCSPI_CHCONF0             0x2c
  49 #define OMAP2_MCSPI_CHSTAT0             0x30
  50 #define OMAP2_MCSPI_CHCTRL0             0x34
  51 #define OMAP2_MCSPI_TX0                 0x38
  52 #define OMAP2_MCSPI_RX0                 0x3c
  53
  54 /* per-register bitmasks: */
  55 #define OMAP2_MCSPI_IRQSTATUS_EOW       BIT(17)
  56
  57 #define OMAP2_MCSPI_MODULCTRL_SINGLE    BIT(0)
  58 #define OMAP2_MCSPI_MODULCTRL_MS        BIT(2)
  59 #define OMAP2_MCSPI_MODULCTRL_STEST     BIT(3)
  60
  61 #define OMAP2_MCSPI_CHCONF_PHA          BIT(0)
  62 #define OMAP2_MCSPI_CHCONF_POL          BIT(1)
  63 #define OMAP2_MCSPI_CHCONF_CLKD_MASK    (0x0f << 2)
  64 #define OMAP2_MCSPI_CHCONF_EPOL         BIT(6)
  65 #define OMAP2_MCSPI_CHCONF_WL_MASK      (0x1f << 7)
  66 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY  BIT(12)
  67 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY  BIT(13)
  68 #define OMAP2_MCSPI_CHCONF_TRM_MASK     (0x03 << 12)
  69 #define OMAP2_MCSPI_CHCONF_DMAW         BIT(14)
  70 #define OMAP2_MCSPI_CHCONF_DMAR         BIT(15)
  71 #define OMAP2_MCSPI_CHCONF_DPE0         BIT(16)
  72 #define OMAP2_MCSPI_CHCONF_DPE1         BIT(17)
  73 #define OMAP2_MCSPI_CHCONF_IS           BIT(18)
  74 #define OMAP2_MCSPI_CHCONF_TURBO        BIT(19)
  75 #define OMAP2_MCSPI_CHCONF_FORCE        BIT(20)
  76 #define OMAP2_MCSPI_CHCONF_FFET         BIT(27)
  77 #define OMAP2_MCSPI_CHCONF_FFER         BIT(28)
  78 #define OMAP2_MCSPI_CHCONF_CLKG         BIT(29)
  79
  80 #define OMAP2_MCSPI_CHSTAT_RXS          BIT(0)
  81 #define OMAP2_MCSPI_CHSTAT_TXS          BIT(1)
  82 #define OMAP2_MCSPI_CHSTAT_EOT          BIT(2)
  83 #define OMAP2_MCSPI_CHSTAT_TXFFE        BIT(3)
  84
  85 #define OMAP2_MCSPI_CHCTRL_EN           BIT(0)
  86 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK  (0xff << 8)
  87
  88 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN   BIT(0)
  89
  90 /* We have 2 DMA channels per CS, one for RX and one for TX */
  91 struct omap2_mcspi_dma {
  92         struct dma_chan *dma_tx;
  93         struct dma_chan *dma_rx;
  94
  95         struct completion dma_tx_completion;
  96         struct completion dma_rx_completion;
  97
  98         char dma_rx_ch_name[14];
  99         char dma_tx_ch_name[14];
 100 };
 101
 102 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 103  * cache operations; better heuristics consider wordsize and bitrate.
 104  */
 105 #define DMA_MIN_BYTES                   160
 106
 107
 108 /*
 109  * Used for context save and restore, structure members to be updated whenever
 110  * corresponding registers are modified.
 111  */
 112 struct omap2_mcspi_regs {
 113         u32 modulctrl;
 114         u32 wakeupenable;
 115         struct list_head cs;
 116 };
 117
 118 struct omap2_mcspi {
 119         struct completion       txdone;
 120         struct spi_master       *master;
 121         /* Virtual base address of the controller */
 122         void __iomem            *base;
 123         unsigned long           phys;
 124         /* SPI1 has 4 channels, while SPI2 has 2 */
 125         struct omap2_mcspi_dma  *dma_channels;
 126         struct device           *dev;
 127         struct omap2_mcspi_regs ctx;
 128         int                     fifo_depth;
 129         bool                    slave_aborted;
 130         unsigned int            pin_dir:1;
 131         size_t                  max_xfer_len;
 132 };
 133
 134 struct omap2_mcspi_cs {
 135         void __iomem            *base;
 136         unsigned long           phys;
 137         int                     word_len;
 138         u16                     mode;
 139         struct list_head        node;
 140         /* Context save and restore shadow register */
 141         u32                     chconf0, chctrl0;
 142 };
 143
 144 static inline void mcspi_write_reg(struct spi_master *master,
 145                 int idx, u32 val)
 146 {
 147         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 148
 149         writel_relaxed(val, mcspi->base + idx);
 150 }
 151
 152 static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
 153 {
 154         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 155
 156         return readl_relaxed(mcspi->base + idx);
 157 }
 158
 159 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 160                 int idx, u32 val)
 161 {
 162         struct omap2_mcspi_cs   *cs = spi->controller_state;
 163
 164         writel_relaxed(val, cs->base +  idx);
 165 }
 166
 167 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 168 {
 169         struct omap2_mcspi_cs   *cs = spi->controller_state;
 170
 171         return readl_relaxed(cs->base + idx);
 172 }
 173
 174 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 175 {
 176         struct omap2_mcspi_cs *cs = spi->controller_state;
 177
 178         return cs->chconf0;
 179 }
 180
 181 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 182 {
 183         struct omap2_mcspi_cs *cs = spi->controller_state;
 184
 185         cs->chconf0 = val;
 186         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 187         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 188 }
 189
 190 static inline int mcspi_bytes_per_word(int word_len)
 191 {
 192         if (word_len <= 8)
 193                 return 1;
 194         else if (word_len <= 16)
 195                 return 2;
 196         else /* word_len <= 32 */
 197                 return 4;
 198 }
 199
 200 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 201                 int is_read, int enable)
 202 {
 203         u32 l, rw;
 204
 205         l = mcspi_cached_chconf0(spi);
 206
 207         if (is_read) /* 1 is read, 0 write */
 208                 rw = OMAP2_MCSPI_CHCONF_DMAR;
 209         else
 210                 rw = OMAP2_MCSPI_CHCONF_DMAW;
 211
 212         if (enable)
 213                 l |= rw;
 214         else
 215                 l &= ~rw;
 216
 217         mcspi_write_chconf0(spi, l);
 218 }
 219
 220 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 221 {
 222         struct omap2_mcspi_cs *cs = spi->controller_state;
 223         u32 l;
 224
 225         l = cs->chctrl0;
 226         if (enable)
 227                 l |= OMAP2_MCSPI_CHCTRL_EN;
 228         else
 229                 l &= ~OMAP2_MCSPI_CHCTRL_EN;
 230         cs->chctrl0 = l;
 231         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 232         /* Flash post-writes */
 233         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 234 }
 235
 236 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 237 {
 238         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 239         u32 l;
 240
 241         /* The controller handles the inverted chip selects
 242          * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 243          * the inversion from the core spi_set_cs function.
 244          */
 245         if (spi->mode & SPI_CS_HIGH)
 246                 enable = !enable;
 247
 248         if (spi->controller_state) {
 249                 int err = pm_runtime_resume_and_get(mcspi->dev);
 250                 if (err < 0) {
 251                         dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 252                         return;
 253                 }
 254
 255                 l = mcspi_cached_chconf0(spi);
 256
 257                 if (enable)
 258                         l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 259                 else
 260                         l |= OMAP2_MCSPI_CHCONF_FORCE;
 261
 262                 mcspi_write_chconf0(spi, l);
 263
 264                 pm_runtime_mark_last_busy(mcspi->dev);
 265                 pm_runtime_put_autosuspend(mcspi->dev);
 266         }
 267 }
 268
 269 static void omap2_mcspi_set_mode(struct spi_master *master)
 270 {
 271         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
 272         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
 273         u32 l;
 274
 275         /*
 276          * Choose master or slave mode
 277          */
 278         l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
 279         l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
 280         if (spi_controller_is_slave(master)) {
 281                 l |= (OMAP2_MCSPI_MODULCTRL_MS);
 282         } else {
 283                 l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
 284                 l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 285         }
 286         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 287
 288         ctx->modulctrl = l;
 289 }
 290
 291 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 292                                 struct spi_transfer *t, int enable)
 293 {
 294         struct spi_master *master = spi->master;
 295         struct omap2_mcspi_cs *cs = spi->controller_state;
 296         struct omap2_mcspi *mcspi;
 297         unsigned int wcnt;
 298         int max_fifo_depth, bytes_per_word;
 299         u32 chconf, xferlevel;
 300
 301         mcspi = spi_master_get_devdata(master);
 302
 303         chconf = mcspi_cached_chconf0(spi);
 304         if (enable) {
 305                 bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 306                 if (t->len % bytes_per_word != 0)
 307                         goto disable_fifo;
 308
 309                 if (t->rx_buf != NULL && t->tx_buf != NULL)
 310                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 311                 else
 312                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 313
 314                 wcnt = t->len / bytes_per_word;
 315                 if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 316                         goto disable_fifo;
 317
 318                 xferlevel = wcnt << 16;
 319                 if (t->rx_buf != NULL) {
 320                         chconf |= OMAP2_MCSPI_CHCONF_FFER;
 321                         xferlevel |= (bytes_per_word - 1) << 8;
 322                 }
 323
 324                 if (t->tx_buf != NULL) {
 325                         chconf |= OMAP2_MCSPI_CHCONF_FFET;
 326                         xferlevel |= bytes_per_word - 1;
 327                 }
 328
 329                 mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 330                 mcspi_write_chconf0(spi, chconf);
 331                 mcspi->fifo_depth = max_fifo_depth;
 332
 333                 return;
 334         }
 335
 336 disable_fifo:
 337         if (t->rx_buf != NULL)
 338                 chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 339
 340         if (t->tx_buf != NULL)
 341                 chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 342
 343         mcspi_write_chconf0(spi, chconf);
 344         mcspi->fifo_depth = 0;
 345 }
 346
 347 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 348 {
 349         unsigned long timeout;
 350
 351         timeout = jiffies + msecs_to_jiffies(1000);
 352         while (!(readl_relaxed(reg) & bit)) {
 353                 if (time_after(jiffies, timeout)) {
 354                         if (!(readl_relaxed(reg) & bit))
 355                                 return -ETIMEDOUT;
 356                         else
 357                                 return 0;
 358                 }
 359                 cpu_relax();
 360         }
 361         return 0;
 362 }
 363
 364 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
 365                                      struct completion *x)
 366 {
 367         if (spi_controller_is_slave(mcspi->master)) {
 368                 if (wait_for_completion_interruptible(x) ||
 369                     mcspi->slave_aborted)
 370                         return -EINTR;
 371         } else {
 372                 wait_for_completion(x);
 373         }
 374
 375         return 0;
 376 }
 377
 378 static void omap2_mcspi_rx_callback(void *data)
 379 {
 380         struct spi_device *spi = data;
 381         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 382         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 383
 384         /* We must disable the DMA RX request */
 385         omap2_mcspi_set_dma_req(spi, 1, 0);
 386
 387         complete(&mcspi_dma->dma_rx_completion);
 388 }
 389
 390 static void omap2_mcspi_tx_callback(void *data)
 391 {
 392         struct spi_device *spi = data;
 393         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 394         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 395
 396         /* We must disable the DMA TX request */
 397         omap2_mcspi_set_dma_req(spi, 0, 0);
 398
 399         complete(&mcspi_dma->dma_tx_completion);
 400 }
 401
 402 static void omap2_mcspi_tx_dma(struct spi_device *spi,
 403                                 struct spi_transfer *xfer,
 404                                 struct dma_slave_config cfg)
 405 {
 406         struct omap2_mcspi      *mcspi;
 407         struct omap2_mcspi_dma  *mcspi_dma;
 408         struct dma_async_tx_descriptor *tx;
 409
 410         mcspi = spi_master_get_devdata(spi->master);
 411         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 412
 413         dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 414
 415         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 416                                      xfer->tx_sg.nents,
 417                                      DMA_MEM_TO_DEV,
 418                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 419         if (tx) {
 420                 tx->callback = omap2_mcspi_tx_callback;
 421                 tx->callback_param = spi;
 422                 dmaengine_submit(tx);
 423         } else {
 424                 /* FIXME: fall back to PIO? */
 425         }
 426         dma_async_issue_pending(mcspi_dma->dma_tx);
 427         omap2_mcspi_set_dma_req(spi, 0, 1);
 428 }
 429
 430 static unsigned
 431 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 432                                 struct dma_slave_config cfg,
 433                                 unsigned es)
 434 {
 435         struct omap2_mcspi      *mcspi;
 436         struct omap2_mcspi_dma  *mcspi_dma;
 437         unsigned int            count, transfer_reduction = 0;
 438         struct scatterlist      *sg_out[2];
 439         int                     nb_sizes = 0, out_mapped_nents[2], ret, x;
 440         size_t                  sizes[2];
 441         u32                     l;
 442         int                     elements = 0;
 443         int                     word_len, element_count;
 444         struct omap2_mcspi_cs   *cs = spi->controller_state;
 445         void __iomem            *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 446         struct dma_async_tx_descriptor *tx;
 447
 448         mcspi = spi_master_get_devdata(spi->master);
 449         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 450         count = xfer->len;
 451
 452         /*
 453          *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 454          *  it mentions reducing DMA transfer length by one element in master
 455          *  normal mode.
 456          */
 457         if (mcspi->fifo_depth == 0)
 458                 transfer_reduction = es;
 459
 460         word_len = cs->word_len;
 461         l = mcspi_cached_chconf0(spi);
 462
 463         if (word_len <= 8)
 464                 element_count = count;
 465         else if (word_len <= 16)
 466                 element_count = count >> 1;
 467         else /* word_len <= 32 */
 468                 element_count = count >> 2;
 469
 470
 471         dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 472
 473         /*
 474          *  Reduce DMA transfer length by one more if McSPI is
 475          *  configured in turbo mode.
 476          */
 477         if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 478                 transfer_reduction += es;
 479
 480         if (transfer_reduction) {
 481                 /* Split sgl into two. The second sgl won't be used. */
 482                 sizes[0] = count - transfer_reduction;
 483                 sizes[1] = transfer_reduction;
 484                 nb_sizes = 2;
 485         } else {
 486                 /*
 487                  * Don't bother splitting the sgl. This essentially
 488                  * clones the original sgl.
 489                  */
 490                 sizes[0] = count;
 491                 nb_sizes = 1;
 492         }
 493
 494         ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
 495                        sizes, sg_out, out_mapped_nents, GFP_KERNEL);
 496
 497         if (ret < 0) {
 498                 dev_err(&spi->dev, "sg_split failed\n");
 499                 return 0;
 500         }
 501
 502         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
 503                                      out_mapped_nents[0], DMA_DEV_TO_MEM,
 504                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 505         if (tx) {
 506                 tx->callback = omap2_mcspi_rx_callback;
 507                 tx->callback_param = spi;
 508                 dmaengine_submit(tx);
 509         } else {
 510                 /* FIXME: fall back to PIO? */
 511         }
 512
 513         dma_async_issue_pending(mcspi_dma->dma_rx);
 514         omap2_mcspi_set_dma_req(spi, 1, 1);
 515
 516         ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
 517         if (ret || mcspi->slave_aborted) {
 518                 dmaengine_terminate_sync(mcspi_dma->dma_rx);
 519                 omap2_mcspi_set_dma_req(spi, 1, 0);
 520                 return 0;
 521         }
 522
 523         for (x = 0; x < nb_sizes; x++)
 524                 kfree(sg_out[x]);
 525
 526         if (mcspi->fifo_depth > 0)
 527                 return count;
 528
 529         /*
 530          *  Due to the DMA transfer length reduction the missing bytes must
 531          *  be read manually to receive all of the expected data.
 532          */
 533         omap2_mcspi_set_enable(spi, 0);
 534
 535         elements = element_count - 1;
 536
 537         if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 538                 elements--;
 539
 540                 if (!mcspi_wait_for_reg_bit(chstat_reg,
 541                                             OMAP2_MCSPI_CHSTAT_RXS)) {
 542                         u32 w;
 543
 544                         w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 545                         if (word_len <= 8)
 546                                 ((u8 *)xfer->rx_buf)[elements++] = w;
 547                         else if (word_len <= 16)
 548                                 ((u16 *)xfer->rx_buf)[elements++] = w;
 549                         else /* word_len <= 32 */
 550                                 ((u32 *)xfer->rx_buf)[elements++] = w;
 551                 } else {
 552                         int bytes_per_word = mcspi_bytes_per_word(word_len);
 553                         dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 554                         count -= (bytes_per_word << 1);
 555                         omap2_mcspi_set_enable(spi, 1);
 556                         return count;
 557                 }
 558         }
 559         if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
 560                 u32 w;
 561
 562                 w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 563                 if (word_len <= 8)
 564                         ((u8 *)xfer->rx_buf)[elements] = w;
 565                 else if (word_len <= 16)
 566                         ((u16 *)xfer->rx_buf)[elements] = w;
 567                 else /* word_len <= 32 */
 568                         ((u32 *)xfer->rx_buf)[elements] = w;
 569         } else {
 570                 dev_err(&spi->dev, "DMA RX last word empty\n");
 571                 count -= mcspi_bytes_per_word(word_len);
 572         }
 573         omap2_mcspi_set_enable(spi, 1);
 574         return count;
 575 }
 576
 577 static unsigned
 578 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 579 {
 580         struct omap2_mcspi      *mcspi;
 581         struct omap2_mcspi_cs   *cs = spi->controller_state;
 582         struct omap2_mcspi_dma  *mcspi_dma;
 583         unsigned int            count;
 584         u8                      *rx;
 585         const u8                *tx;
 586         struct dma_slave_config cfg;
 587         enum dma_slave_buswidth width;
 588         unsigned es;
 589         void __iomem            *chstat_reg;
 590         void __iomem            *irqstat_reg;
 591         int                     wait_res;
 592
 593         mcspi = spi_master_get_devdata(spi->master);
 594         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 595
 596         if (cs->word_len <= 8) {
 597                 width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 598                 es = 1;
 599         } else if (cs->word_len <= 16) {
 600                 width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 601                 es = 2;
 602         } else {
 603                 width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 604                 es = 4;
 605         }
 606
 607         count = xfer->len;
 608
 609         memset(&cfg, 0, sizeof(cfg));
 610         cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 611         cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 612         cfg.src_addr_width = width;
 613         cfg.dst_addr_width = width;
 614         cfg.src_maxburst = 1;
 615         cfg.dst_maxburst = 1;
 616
 617         rx = xfer->rx_buf;
 618         tx = xfer->tx_buf;
 619
 620         mcspi->slave_aborted = false;
 621         reinit_completion(&mcspi_dma->dma_tx_completion);
 622         reinit_completion(&mcspi_dma->dma_rx_completion);
 623         reinit_completion(&mcspi->txdone);
 624         if (tx) {
 625                 /* Enable EOW IRQ to know end of tx in slave mode */
 626                 if (spi_controller_is_slave(spi->master))
 627                         mcspi_write_reg(spi->master,
 628                                         OMAP2_MCSPI_IRQENABLE,
 629                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 630                 omap2_mcspi_tx_dma(spi, xfer, cfg);
 631         }
 632
 633         if (rx != NULL)
 634                 count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 635
 636         if (tx != NULL) {
 637                 int ret;
 638
 639                 ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
 640                 if (ret || mcspi->slave_aborted) {
 641                         dmaengine_terminate_sync(mcspi_dma->dma_tx);
 642                         omap2_mcspi_set_dma_req(spi, 0, 0);
 643                         return 0;
 644                 }
 645
 646                 if (spi_controller_is_slave(mcspi->master)) {
 647                         ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
 648                         if (ret || mcspi->slave_aborted)
 649                                 return 0;
 650                 }
 651
 652                 if (mcspi->fifo_depth > 0) {
 653                         irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 654
 655                         if (mcspi_wait_for_reg_bit(irqstat_reg,
 656                                                 OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 657                                 dev_err(&spi->dev, "EOW timed out\n");
 658
 659                         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
 660                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 661                 }
 662
 663                 /* for TX_ONLY mode, be sure all words have shifted out */
 664                 if (rx == NULL) {
 665                         chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 666                         if (mcspi->fifo_depth > 0) {
 667                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 668                                                 OMAP2_MCSPI_CHSTAT_TXFFE);
 669                                 if (wait_res < 0)
 670                                         dev_err(&spi->dev, "TXFFE timed out\n");
 671                         } else {
 672                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 673                                                 OMAP2_MCSPI_CHSTAT_TXS);
 674                                 if (wait_res < 0)
 675                                         dev_err(&spi->dev, "TXS timed out\n");
 676                         }
 677                         if (wait_res >= 0 &&
 678                                 (mcspi_wait_for_reg_bit(chstat_reg,
 679                                         OMAP2_MCSPI_CHSTAT_EOT) < 0))
 680                                 dev_err(&spi->dev, "EOT timed out\n");
 681                 }
 682         }
 683         return count;
 684 }
 685
 686 static unsigned
 687 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 688 {
 689         struct omap2_mcspi_cs   *cs = spi->controller_state;
 690         unsigned int            count, c;
 691         u32                     l;
 692         void __iomem            *base = cs->base;
 693         void __iomem            *tx_reg;
 694         void __iomem            *rx_reg;
 695         void __iomem            *chstat_reg;
 696         int                     word_len;
 697
 698         count = xfer->len;
 699         c = count;
 700         word_len = cs->word_len;
 701
 702         l = mcspi_cached_chconf0(spi);
 703
 704         /* We store the pre-calculated register addresses on stack to speed
 705          * up the transfer loop. */
 706         tx_reg          = base + OMAP2_MCSPI_TX0;
 707         rx_reg          = base + OMAP2_MCSPI_RX0;
 708         chstat_reg      = base + OMAP2_MCSPI_CHSTAT0;
 709
 710         if (c < (word_len>>3))
 711                 return 0;
 712
 713         if (word_len <= 8) {
 714                 u8              *rx;
 715                 const u8        *tx;
 716
 717                 rx = xfer->rx_buf;
 718                 tx = xfer->tx_buf;
 719
 720                 do {
 721                         c -= 1;
 722                         if (tx != NULL) {
 723                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 724                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 725                                         dev_err(&spi->dev, "TXS timed out\n");
 726                                         goto out;
 727                                 }
 728                                 dev_vdbg(&spi->dev, "write-%d %02x\n",
 729                                                 word_len, *tx);
 730                                 writel_relaxed(*tx++, tx_reg);
 731                         }
 732                         if (rx != NULL) {
 733                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 734                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 735                                         dev_err(&spi->dev, "RXS timed out\n");
 736                                         goto out;
 737                                 }
 738
 739                                 if (c == 1 && tx == NULL &&
 740                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 741                                         omap2_mcspi_set_enable(spi, 0);
 742                                         *rx++ = readl_relaxed(rx_reg);
 743                                         dev_vdbg(&spi->dev, "read-%d %02x\n",
 744                                                     word_len, *(rx - 1));
 745                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 746                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 747                                                 dev_err(&spi->dev,
 748                                                         "RXS timed out\n");
 749                                                 goto out;
 750                                         }
 751                                         c = 0;
 752                                 } else if (c == 0 && tx == NULL) {
 753                                         omap2_mcspi_set_enable(spi, 0);
 754                                 }
 755
 756                                 *rx++ = readl_relaxed(rx_reg);
 757                                 dev_vdbg(&spi->dev, "read-%d %02x\n",
 758                                                 word_len, *(rx - 1));
 759                         }
 760                         /* Add word delay between each word */
 761                         spi_delay_exec(&xfer->word_delay, xfer);
 762                 } while (c);
 763         } else if (word_len <= 16) {
 764                 u16             *rx;
 765                 const u16       *tx;
 766
 767                 rx = xfer->rx_buf;
 768                 tx = xfer->tx_buf;
 769                 do {
 770                         c -= 2;
 771                         if (tx != NULL) {
 772                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 773                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 774                                         dev_err(&spi->dev, "TXS timed out\n");
 775                                         goto out;
 776                                 }
 777                                 dev_vdbg(&spi->dev, "write-%d %04x\n",
 778                                                 word_len, *tx);
 779                                 writel_relaxed(*tx++, tx_reg);
 780                         }
 781                         if (rx != NULL) {
 782                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 783                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 784                                         dev_err(&spi->dev, "RXS timed out\n");
 785                                         goto out;
 786                                 }
 787
 788                                 if (c == 2 && tx == NULL &&
 789                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 790                                         omap2_mcspi_set_enable(spi, 0);
 791                                         *rx++ = readl_relaxed(rx_reg);
 792                                         dev_vdbg(&spi->dev, "read-%d %04x\n",
 793                                                     word_len, *(rx - 1));
 794                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 795                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 796                                                 dev_err(&spi->dev,
 797                                                         "RXS timed out\n");
 798                                                 goto out;
 799                                         }
 800                                         c = 0;
 801                                 } else if (c == 0 && tx == NULL) {
 802                                         omap2_mcspi_set_enable(spi, 0);
 803                                 }
 804
 805                                 *rx++ = readl_relaxed(rx_reg);
 806                                 dev_vdbg(&spi->dev, "read-%d %04x\n",
 807                                                 word_len, *(rx - 1));
 808                         }
 809                         /* Add word delay between each word */
 810                         spi_delay_exec(&xfer->word_delay, xfer);
 811                 } while (c >= 2);
 812         } else if (word_len <= 32) {
 813                 u32             *rx;
 814                 const u32       *tx;
 815
 816                 rx = xfer->rx_buf;
 817                 tx = xfer->tx_buf;
 818                 do {
 819                         c -= 4;
 820                         if (tx != NULL) {
 821                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 822                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 823                                         dev_err(&spi->dev, "TXS timed out\n");
 824                                         goto out;
 825                                 }
 826                                 dev_vdbg(&spi->dev, "write-%d %08x\n",
 827                                                 word_len, *tx);
 828                                 writel_relaxed(*tx++, tx_reg);
 829                         }
 830                         if (rx != NULL) {
 831                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 832                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 833                                         dev_err(&spi->dev, "RXS timed out\n");
 834                                         goto out;
 835                                 }
 836
 837                                 if (c == 4 && tx == NULL &&
 838                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 839                                         omap2_mcspi_set_enable(spi, 0);
 840                                         *rx++ = readl_relaxed(rx_reg);
 841                                         dev_vdbg(&spi->dev, "read-%d %08x\n",
 842                                                     word_len, *(rx - 1));
 843                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 844                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 845                                                 dev_err(&spi->dev,
 846                                                         "RXS timed out\n");
 847                                                 goto out;
 848                                         }
 849                                         c = 0;
 850                                 } else if (c == 0 && tx == NULL) {
 851                                         omap2_mcspi_set_enable(spi, 0);
 852                                 }
 853
 854                                 *rx++ = readl_relaxed(rx_reg);
 855                                 dev_vdbg(&spi->dev, "read-%d %08x\n",
 856                                                 word_len, *(rx - 1));
 857                         }
 858                         /* Add word delay between each word */
 859                         spi_delay_exec(&xfer->word_delay, xfer);
 860                 } while (c >= 4);
 861         }
 862
 863         /* for TX_ONLY mode, be sure all words have shifted out */
 864         if (xfer->rx_buf == NULL) {
 865                 if (mcspi_wait_for_reg_bit(chstat_reg,
 866                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 867                         dev_err(&spi->dev, "TXS timed out\n");
 868                 } else if (mcspi_wait_for_reg_bit(chstat_reg,
 869                                 OMAP2_MCSPI_CHSTAT_EOT) < 0)
 870                         dev_err(&spi->dev, "EOT timed out\n");
 871
 872                 /* disable chan to purge rx datas received in TX_ONLY transfer,
 873                  * otherwise these rx datas will affect the direct following
 874                  * RX_ONLY transfer.
 875                  */
 876                 omap2_mcspi_set_enable(spi, 0);
 877         }
 878 out:
 879         omap2_mcspi_set_enable(spi, 1);
 880         return count - c;
 881 }
 882
 883 static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
 884 {
 885         u32 div;
 886
 887         for (div = 0; div < 15; div++)
 888                 if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
 889                         return div;
 890
 891         return 15;
 892 }
 893
 894 /* called only when no transfer is active to this device */
 895 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 896                 struct spi_transfer *t)
 897 {
 898         struct omap2_mcspi_cs *cs = spi->controller_state;
 899         struct omap2_mcspi *mcspi;
 900         u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 901         u8 word_len = spi->bits_per_word;
 902         u32 speed_hz = spi->max_speed_hz;
 903
 904         mcspi = spi_master_get_devdata(spi->master);
 905
 906         if (t != NULL && t->bits_per_word)
 907                 word_len = t->bits_per_word;
 908
 909         cs->word_len = word_len;
 910
 911         if (t && t->speed_hz)
 912                 speed_hz = t->speed_hz;
 913
 914         speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
 915         if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
 916                 clkd = omap2_mcspi_calc_divisor(speed_hz);
 917                 speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
 918                 clkg = 0;
 919         } else {
 920                 div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
 921                 speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
 922                 clkd = (div - 1) & 0xf;
 923                 extclk = (div - 1) >> 4;
 924                 clkg = OMAP2_MCSPI_CHCONF_CLKG;
 925         }
 926
 927         l = mcspi_cached_chconf0(spi);
 928
 929         /* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
 930          * REVISIT: this controller could support SPI_3WIRE mode.
 931          */
 932         if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 933                 l &= ~OMAP2_MCSPI_CHCONF_IS;
 934                 l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 935                 l |= OMAP2_MCSPI_CHCONF_DPE0;
 936         } else {
 937                 l |= OMAP2_MCSPI_CHCONF_IS;
 938                 l |= OMAP2_MCSPI_CHCONF_DPE1;
 939                 l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 940         }
 941
 942         /* wordlength */
 943         l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 944         l |= (word_len - 1) << 7;
 945
 946         /* set chipselect polarity; manage with FORCE */
 947         if (!(spi->mode & SPI_CS_HIGH))
 948                 l |= OMAP2_MCSPI_CHCONF_EPOL;   /* active-low; normal */
 949         else
 950                 l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 951
 952         /* set clock divisor */
 953         l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 954         l |= clkd << 2;
 955
 956         /* set clock granularity */
 957         l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 958         l |= clkg;
 959         if (clkg) {
 960                 cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 961                 cs->chctrl0 |= extclk << 8;
 962                 mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 963         }
 964
 965         /* set SPI mode 0..3 */
 966         if (spi->mode & SPI_CPOL)
 967                 l |= OMAP2_MCSPI_CHCONF_POL;
 968         else
 969                 l &= ~OMAP2_MCSPI_CHCONF_POL;
 970         if (spi->mode & SPI_CPHA)
 971                 l |= OMAP2_MCSPI_CHCONF_PHA;
 972         else
 973                 l &= ~OMAP2_MCSPI_CHCONF_PHA;
 974
 975         mcspi_write_chconf0(spi, l);
 976
 977         cs->mode = spi->mode;
 978
 979         dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 980                         speed_hz,
 981                         (spi->mode & SPI_CPHA) ? "trailing" : "leading",
 982                         (spi->mode & SPI_CPOL) ? "inverted" : "normal");
 983
 984         return 0;
 985 }
 986
 987 /*
 988  * Note that we currently allow DMA only if we get a channel
 989  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
 990  */
 991 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
 992                                    struct omap2_mcspi_dma *mcspi_dma)
 993 {
 994         int ret = 0;
 995
 996         mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
 997                                              mcspi_dma->dma_rx_ch_name);
 998         if (IS_ERR(mcspi_dma->dma_rx)) {
 999                 ret = PTR_ERR(mcspi_dma->dma_rx);
1000                 mcspi_dma->dma_rx = NULL;
1001                 goto no_dma;
1002         }
1003
1004         mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
1005                                              mcspi_dma->dma_tx_ch_name);
1006         if (IS_ERR(mcspi_dma->dma_tx)) {
1007                 ret = PTR_ERR(mcspi_dma->dma_tx);
1008                 mcspi_dma->dma_tx = NULL;
1009                 dma_release_channel(mcspi_dma->dma_rx);
1010                 mcspi_dma->dma_rx = NULL;
1011         }
1012
1013         init_completion(&mcspi_dma->dma_rx_completion);
1014         init_completion(&mcspi_dma->dma_tx_completion);
1015
1016 no_dma:
1017         return ret;
1018 }
1019
1020 static void omap2_mcspi_release_dma(struct spi_master *master)
1021 {
1022         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1023         struct omap2_mcspi_dma  *mcspi_dma;
1024         int i;
1025
1026         for (i = 0; i < master->num_chipselect; i++) {
1027                 mcspi_dma = &mcspi->dma_channels[i];
1028
1029                 if (mcspi_dma->dma_rx) {
1030                         dma_release_channel(mcspi_dma->dma_rx);
1031                         mcspi_dma->dma_rx = NULL;
1032                 }
1033                 if (mcspi_dma->dma_tx) {
1034                         dma_release_channel(mcspi_dma->dma_tx);
1035                         mcspi_dma->dma_tx = NULL;
1036                 }
1037         }
1038 }
1039
1040 static void omap2_mcspi_cleanup(struct spi_device *spi)
1041 {
1042         struct omap2_mcspi_cs   *cs;
1043
1044         if (spi->controller_state) {
1045                 /* Unlink controller state from context save list */
1046                 cs = spi->controller_state;
1047                 list_del(&cs->node);
1048
1049                 kfree(cs);
1050         }
1051 }
1052
1053 static int omap2_mcspi_setup(struct spi_device *spi)
1054 {
1055         bool                    initial_setup = false;
1056         int                     ret;
1057         struct omap2_mcspi      *mcspi = spi_master_get_devdata(spi->master);
1058         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1059         struct omap2_mcspi_cs   *cs = spi->controller_state;
1060
1061         if (!cs) {
1062                 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
1063                 if (!cs)
1064                         return -ENOMEM;
1065                 cs->base = mcspi->base + spi->chip_select * 0x14;
1066                 cs->phys = mcspi->phys + spi->chip_select * 0x14;
1067                 cs->mode = 0;
1068                 cs->chconf0 = 0;
1069                 cs->chctrl0 = 0;
1070                 spi->controller_state = cs;
1071                 /* Link this to context save list */
1072                 list_add_tail(&cs->node, &ctx->cs);
1073                 initial_setup = true;
1074         }
1075
1076         ret = pm_runtime_resume_and_get(mcspi->dev);
1077         if (ret < 0) {
1078                 if (initial_setup)
1079                         omap2_mcspi_cleanup(spi);
1080
1081                 return ret;
1082         }
1083
1084         ret = omap2_mcspi_setup_transfer(spi, NULL);
1085         if (ret && initial_setup)
1086                 omap2_mcspi_cleanup(spi);
1087
1088         pm_runtime_mark_last_busy(mcspi->dev);
1089         pm_runtime_put_autosuspend(mcspi->dev);
1090
1091         return ret;
1092 }
1093
1094 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1095 {
1096         struct omap2_mcspi *mcspi = data;
1097         u32 irqstat;
1098
1099         irqstat = mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
1100         if (!irqstat)
1101                 return IRQ_NONE;
1102
1103         /* Disable IRQ and wakeup slave xfer task */
1104         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
1105         if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1106                 complete(&mcspi->txdone);
1107
1108         return IRQ_HANDLED;
1109 }
1110
1111 static int omap2_mcspi_slave_abort(struct spi_master *master)
1112 {
1113         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1114         struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1115
1116         mcspi->slave_aborted = true;
1117         complete(&mcspi_dma->dma_rx_completion);
1118         complete(&mcspi_dma->dma_tx_completion);
1119         complete(&mcspi->txdone);
1120
1121         return 0;
1122 }
1123
1124 static int omap2_mcspi_transfer_one(struct spi_master *master,
1125                                     struct spi_device *spi,
1126                                     struct spi_transfer *t)
1127 {
1128
1129         /* We only enable one channel at a time -- the one whose message is
1130          * -- although this controller would gladly
1131          * arbitrate among multiple channels.  This corresponds to "single
1132          * channel" master mode.  As a side effect, we need to manage the
1133          * chipselect with the FORCE bit ... CS != channel enable.
1134          */
1135
1136         struct omap2_mcspi              *mcspi;
1137         struct omap2_mcspi_dma          *mcspi_dma;
1138         struct omap2_mcspi_cs           *cs;
1139         struct omap2_mcspi_device_config *cd;
1140         int                             par_override = 0;
1141         int                             status = 0;
1142         u32                             chconf;
1143
1144         mcspi = spi_master_get_devdata(master);
1145         mcspi_dma = mcspi->dma_channels + spi->chip_select;
1146         cs = spi->controller_state;
1147         cd = spi->controller_data;
1148
1149         /*
1150          * The slave driver could have changed spi->mode in which case
1151          * it will be different from cs->mode (the current hardware setup).
1152          * If so, set par_override (even though its not a parity issue) so
1153          * omap2_mcspi_setup_transfer will be called to configure the hardware
1154          * with the correct mode on the first iteration of the loop below.
1155          */
1156         if (spi->mode != cs->mode)
1157                 par_override = 1;
1158
1159         omap2_mcspi_set_enable(spi, 0);
1160
1161         if (spi->cs_gpiod)
1162                 omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1163
1164         if (par_override ||
1165             (t->speed_hz != spi->max_speed_hz) ||
1166             (t->bits_per_word != spi->bits_per_word)) {
1167                 par_override = 1;
1168                 status = omap2_mcspi_setup_transfer(spi, t);
1169                 if (status < 0)
1170                         goto out;
1171                 if (t->speed_hz == spi->max_speed_hz &&
1172                     t->bits_per_word == spi->bits_per_word)
1173                         par_override = 0;
1174         }
1175         if (cd && cd->cs_per_word) {
1176                 chconf = mcspi->ctx.modulctrl;
1177                 chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1178                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1179                 mcspi->ctx.modulctrl =
1180                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1181         }
1182
1183         chconf = mcspi_cached_chconf0(spi);
1184         chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1185         chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1186
1187         if (t->tx_buf == NULL)
1188                 chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1189         else if (t->rx_buf == NULL)
1190                 chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1191
1192         if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1193                 /* Turbo mode is for more than one word */
1194                 if (t->len > ((cs->word_len + 7) >> 3))
1195                         chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1196         }
1197
1198         mcspi_write_chconf0(spi, chconf);
1199
1200         if (t->len) {
1201                 unsigned        count;
1202
1203                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1204                     master->cur_msg_mapped &&
1205                     master->can_dma(master, spi, t))
1206                         omap2_mcspi_set_fifo(spi, t, 1);
1207
1208                 omap2_mcspi_set_enable(spi, 1);
1209
1210                 /* RX_ONLY mode needs dummy data in TX reg */
1211                 if (t->tx_buf == NULL)
1212                         writel_relaxed(0, cs->base
1213                                         + OMAP2_MCSPI_TX0);
1214
1215                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1216                     master->cur_msg_mapped &&
1217                     master->can_dma(master, spi, t))
1218                         count = omap2_mcspi_txrx_dma(spi, t);
1219                 else
1220                         count = omap2_mcspi_txrx_pio(spi, t);
1221
1222                 if (count != t->len) {
1223                         status = -EIO;
1224                         goto out;
1225                 }
1226         }
1227
1228         omap2_mcspi_set_enable(spi, 0);
1229
1230         if (mcspi->fifo_depth > 0)
1231                 omap2_mcspi_set_fifo(spi, t, 0);
1232
1233 out:
1234         /* Restore defaults if they were overriden */
1235         if (par_override) {
1236                 par_override = 0;
1237                 status = omap2_mcspi_setup_transfer(spi, NULL);
1238         }
1239
1240         if (cd && cd->cs_per_word) {
1241                 chconf = mcspi->ctx.modulctrl;
1242                 chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1243                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1244                 mcspi->ctx.modulctrl =
1245                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1246         }
1247
1248         omap2_mcspi_set_enable(spi, 0);
1249
1250         if (spi->cs_gpiod)
1251                 omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1252
1253         if (mcspi->fifo_depth > 0 && t)
1254                 omap2_mcspi_set_fifo(spi, t, 0);
1255
1256         return status;
1257 }
1258
1259 static int omap2_mcspi_prepare_message(struct spi_master *master,
1260                                        struct spi_message *msg)
1261 {
1262         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
1263         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1264         struct omap2_mcspi_cs   *cs;
1265
1266         /* Only a single channel can have the FORCE bit enabled
1267          * in its chconf0 register.
1268          * Scan all channels and disable them except the current one.
1269          * A FORCE can remain from a last transfer having cs_change enabled
1270          */
1271         list_for_each_entry(cs, &ctx->cs, node) {
1272                 if (msg->spi->controller_state == cs)
1273                         continue;
1274
1275                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1276                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1277                         writel_relaxed(cs->chconf0,
1278                                         cs->base + OMAP2_MCSPI_CHCONF0);
1279                         readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1280                 }
1281         }
1282
1283         return 0;
1284 }
1285
1286 static bool omap2_mcspi_can_dma(struct spi_master *master,
1287                                 struct spi_device *spi,
1288                                 struct spi_transfer *xfer)
1289 {
1290         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1291         struct omap2_mcspi_dma *mcspi_dma =
1292                 &mcspi->dma_channels[spi->chip_select];
1293
1294         if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1295                 return false;
1296
1297         if (spi_controller_is_slave(master))
1298                 return true;
1299
1300         master->dma_rx = mcspi_dma->dma_rx;
1301         master->dma_tx = mcspi_dma->dma_tx;
1302
1303         return (xfer->len >= DMA_MIN_BYTES);
1304 }
1305
1306 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1307 {
1308         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1309         struct omap2_mcspi_dma *mcspi_dma =
1310                 &mcspi->dma_channels[spi->chip_select];
1311
1312         if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1313                 return mcspi->max_xfer_len;
1314
1315         return SIZE_MAX;
1316 }
1317
1318 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1319 {
1320         struct spi_master       *master = mcspi->master;
1321         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1322         int                     ret = 0;
1323
1324         ret = pm_runtime_resume_and_get(mcspi->dev);
1325         if (ret < 0)
1326                 return ret;
1327
1328         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1329                         OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1330         ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1331
1332         omap2_mcspi_set_mode(master);
1333         pm_runtime_mark_last_busy(mcspi->dev);
1334         pm_runtime_put_autosuspend(mcspi->dev);
1335         return 0;
1336 }
1337
1338 static int omap_mcspi_runtime_suspend(struct device *dev)
1339 {
1340         int error;
1341
1342         error = pinctrl_pm_select_idle_state(dev);
1343         if (error)
1344                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1345
1346         return 0;
1347 }
1348
1349 /*
1350  * When SPI wake up from off-mode, CS is in activate state. If it was in
1351  * inactive state when driver was suspend, then force it to inactive state at
1352  * wake up.
1353  */
1354 static int omap_mcspi_runtime_resume(struct device *dev)
1355 {
1356         struct spi_master *master = dev_get_drvdata(dev);
1357         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1358         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1359         struct omap2_mcspi_cs *cs;
1360         int error;
1361
1362         error = pinctrl_pm_select_default_state(dev);
1363         if (error)
1364                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1365
1366         /* McSPI: context restore */
1367         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1368         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1369
1370         list_for_each_entry(cs, &ctx->cs, node) {
1371                 /*
1372                  * We need to toggle CS state for OMAP take this
1373                  * change in account.
1374                  */
1375                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1376                         cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1377                         writel_relaxed(cs->chconf0,
1378                                        cs->base + OMAP2_MCSPI_CHCONF0);
1379                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1380                         writel_relaxed(cs->chconf0,
1381                                        cs->base + OMAP2_MCSPI_CHCONF0);
1382                 } else {
1383                         writel_relaxed(cs->chconf0,
1384                                        cs->base + OMAP2_MCSPI_CHCONF0);
1385                 }
1386         }
1387
1388         return 0;
1389 }
1390
1391 static struct omap2_mcspi_platform_config omap2_pdata = {
1392         .regs_offset = 0,
1393 };
1394
1395 static struct omap2_mcspi_platform_config omap4_pdata = {
1396         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1397 };
1398
1399 static struct omap2_mcspi_platform_config am654_pdata = {
1400         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1401         .max_xfer_len = SZ_4K - 1,
1402 };
1403
1404 static const struct of_device_id omap_mcspi_of_match[] = {
1405         {
1406                 .compatible = "ti,omap2-mcspi",
1407                 .data = &omap2_pdata,
1408         },
1409         {
1410                 .compatible = "ti,omap4-mcspi",
1411                 .data = &omap4_pdata,
1412         },
1413         {
1414                 .compatible = "ti,am654-mcspi",
1415                 .data = &am654_pdata,
1416         },
1417         { },
1418 };
1419 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1420
1421 static int omap2_mcspi_probe(struct platform_device *pdev)
1422 {
1423         struct spi_master       *master;
1424         const struct omap2_mcspi_platform_config *pdata;
1425         struct omap2_mcspi      *mcspi;
1426         struct resource         *r;
1427         int                     status = 0, i;
1428         u32                     regs_offset = 0;
1429         struct device_node      *node = pdev->dev.of_node;
1430         const struct of_device_id *match;
1431
1432         if (of_property_read_bool(node, "spi-slave"))
1433                 master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
1434         else
1435                 master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
1436         if (!master)
1437                 return -ENOMEM;
1438
1439         /* the spi->mode bits understood by this driver: */
1440         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1441         master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1442         master->setup = omap2_mcspi_setup;
1443         master->auto_runtime_pm = true;
1444         master->prepare_message = omap2_mcspi_prepare_message;
1445         master->can_dma = omap2_mcspi_can_dma;
1446         master->transfer_one = omap2_mcspi_transfer_one;
1447         master->set_cs = omap2_mcspi_set_cs;
1448         master->cleanup = omap2_mcspi_cleanup;
1449         master->slave_abort = omap2_mcspi_slave_abort;
1450         master->dev.of_node = node;
1451         master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1452         master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1453         master->use_gpio_descriptors = true;
1454
1455         platform_set_drvdata(pdev, master);
1456
1457         mcspi = spi_master_get_devdata(master);
1458         mcspi->master = master;
1459
1460         match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1461         if (match) {
1462                 u32 num_cs = 1; /* default number of chipselect */
1463                 pdata = match->data;
1464
1465                 of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1466                 master->num_chipselect = num_cs;
1467                 if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1468                         mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1469         } else {
1470                 pdata = dev_get_platdata(&pdev->dev);
1471                 master->num_chipselect = pdata->num_cs;
1472                 mcspi->pin_dir = pdata->pin_dir;
1473         }
1474         regs_offset = pdata->regs_offset;
1475         if (pdata->max_xfer_len) {
1476                 mcspi->max_xfer_len = pdata->max_xfer_len;
1477                 master->max_transfer_size = omap2_mcspi_max_xfer_size;
1478         }
1479
1480         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1481         mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1482         if (IS_ERR(mcspi->base)) {
1483                 status = PTR_ERR(mcspi->base);
1484                 goto free_master;
1485         }
1486         mcspi->phys = r->start + regs_offset;
1487         mcspi->base += regs_offset;
1488
1489         mcspi->dev = &pdev->dev;
1490
1491         INIT_LIST_HEAD(&mcspi->ctx.cs);
1492
1493         mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1494                                            sizeof(struct omap2_mcspi_dma),
1495                                            GFP_KERNEL);
1496         if (mcspi->dma_channels == NULL) {
1497                 status = -ENOMEM;
1498                 goto free_master;
1499         }
1500
1501         for (i = 0; i < master->num_chipselect; i++) {
1502                 sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1503                 sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1504
1505                 status = omap2_mcspi_request_dma(mcspi,
1506                                                  &mcspi->dma_channels[i]);
1507                 if (status == -EPROBE_DEFER)
1508                         goto free_master;
1509         }
1510
1511         status = platform_get_irq(pdev, 0);
1512         if (status == -EPROBE_DEFER)
1513                 goto free_master;
1514         if (status < 0) {
1515                 dev_err(&pdev->dev, "no irq resource found\n");
1516                 goto free_master;
1517         }
1518         init_completion(&mcspi->txdone);
1519         status = devm_request_irq(&pdev->dev, status,
1520                                   omap2_mcspi_irq_handler, 0, pdev->name,
1521                                   mcspi);
1522         if (status) {
1523                 dev_err(&pdev->dev, "Cannot request IRQ");
1524                 goto free_master;
1525         }
1526
1527         pm_runtime_use_autosuspend(&pdev->dev);
1528         pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1529         pm_runtime_enable(&pdev->dev);
1530
1531         status = omap2_mcspi_controller_setup(mcspi);
1532         if (status < 0)
1533                 goto disable_pm;
1534
1535         status = devm_spi_register_controller(&pdev->dev, master);
1536         if (status < 0)
1537                 goto disable_pm;
1538
1539         return status;
1540
1541 disable_pm:
1542         pm_runtime_dont_use_autosuspend(&pdev->dev);
1543         pm_runtime_put_sync(&pdev->dev);
1544         pm_runtime_disable(&pdev->dev);
1545 free_master:
1546         omap2_mcspi_release_dma(master);
1547         spi_master_put(master);
1548         return status;
1549 }
1550
1551 static int omap2_mcspi_remove(struct platform_device *pdev)
1552 {
1553         struct spi_master *master = platform_get_drvdata(pdev);
1554         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1555
1556         omap2_mcspi_release_dma(master);
1557
1558         pm_runtime_dont_use_autosuspend(mcspi->dev);
1559         pm_runtime_put_sync(mcspi->dev);
1560         pm_runtime_disable(&pdev->dev);
1561
1562         return 0;
1563 }
1564
1565 /* work with hotplug and coldplug */
1566 MODULE_ALIAS("platform:omap2_mcspi");
1567
1568 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1569 {
1570         struct spi_master *master = dev_get_drvdata(dev);
1571         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1572         int error;
1573
1574         error = pinctrl_pm_select_sleep_state(dev);
1575         if (error)
1576                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1577                          __func__, error);
1578
1579         error = spi_master_suspend(master);
1580         if (error)
1581                 dev_warn(mcspi->dev, "%s: master suspend failed: %i\n",
1582                          __func__, error);
1583
1584         return pm_runtime_force_suspend(dev);
1585 }
1586
1587 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1588 {
1589         struct spi_master *master = dev_get_drvdata(dev);
1590         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1591         int error;
1592
1593         error = spi_master_resume(master);
1594         if (error)
1595                 dev_warn(mcspi->dev, "%s: master resume failed: %i\n",
1596                          __func__, error);
1597
1598         return pm_runtime_force_resume(dev);
1599 }
1600
1601 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1602         SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1603                                 omap2_mcspi_resume)
1604         .runtime_suspend        = omap_mcspi_runtime_suspend,
1605         .runtime_resume         = omap_mcspi_runtime_resume,
1606 };
1607
1608 static struct platform_driver omap2_mcspi_driver = {
1609         .driver = {
1610                 .name =         "omap2_mcspi",
1611                 .pm =           &omap2_mcspi_pm_ops,
1612                 .of_match_table = omap_mcspi_of_match,
1613         },
1614         .probe =        omap2_mcspi_probe,
1615         .remove =       omap2_mcspi_remove,
1616 };
1617
1618 module_platform_driver(omap2_mcspi_driver);
1619 MODULE_LICENSE("GPL");