drivers/spi/spi-omap2-mcspi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * OMAP2 McSPI controller driver
   4  *
   5  * Copyright (C) 2005, 2006 Nokia Corporation
   6  * Author:      Samuel Ortiz <samuel.ortiz@nokia.com> and
   7  *              Juha Yrjola <juha.yrjola@nokia.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/device.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/err.h>
  20 #include <linux/clk.h>
  21 #include <linux/io.h>
  22 #include <linux/slab.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/of.h>
  25 #include <linux/of_device.h>
  26 #include <linux/gcd.h>
  27
  28 #include <linux/spi/spi.h>
  29
  30 #include <linux/platform_data/spi-omap2-mcspi.h>
  31
  32 #define OMAP2_MCSPI_MAX_FREQ            48000000
  33 #define OMAP2_MCSPI_MAX_DIVIDER         4096
  34 #define OMAP2_MCSPI_MAX_FIFODEPTH       64
  35 #define OMAP2_MCSPI_MAX_FIFOWCNT        0xFFFF
  36 #define SPI_AUTOSUSPEND_TIMEOUT         2000
  37
  38 #define OMAP2_MCSPI_REVISION            0x00
  39 #define OMAP2_MCSPI_SYSSTATUS           0x14
  40 #define OMAP2_MCSPI_IRQSTATUS           0x18
  41 #define OMAP2_MCSPI_IRQENABLE           0x1c
  42 #define OMAP2_MCSPI_WAKEUPENABLE        0x20
  43 #define OMAP2_MCSPI_SYST                0x24
  44 #define OMAP2_MCSPI_MODULCTRL           0x28
  45 #define OMAP2_MCSPI_XFERLEVEL           0x7c
  46
  47 /* per-channel banks, 0x14 bytes each, first is: */
  48 #define OMAP2_MCSPI_CHCONF0             0x2c
  49 #define OMAP2_MCSPI_CHSTAT0             0x30
  50 #define OMAP2_MCSPI_CHCTRL0             0x34
  51 #define OMAP2_MCSPI_TX0                 0x38
  52 #define OMAP2_MCSPI_RX0                 0x3c
  53
  54 /* per-register bitmasks: */
  55 #define OMAP2_MCSPI_IRQSTATUS_EOW       BIT(17)
  56
  57 #define OMAP2_MCSPI_MODULCTRL_SINGLE    BIT(0)
  58 #define OMAP2_MCSPI_MODULCTRL_MS        BIT(2)
  59 #define OMAP2_MCSPI_MODULCTRL_STEST     BIT(3)
  60
  61 #define OMAP2_MCSPI_CHCONF_PHA          BIT(0)
  62 #define OMAP2_MCSPI_CHCONF_POL          BIT(1)
  63 #define OMAP2_MCSPI_CHCONF_CLKD_MASK    (0x0f << 2)
  64 #define OMAP2_MCSPI_CHCONF_EPOL         BIT(6)
  65 #define OMAP2_MCSPI_CHCONF_WL_MASK      (0x1f << 7)
  66 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY  BIT(12)
  67 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY  BIT(13)
  68 #define OMAP2_MCSPI_CHCONF_TRM_MASK     (0x03 << 12)
  69 #define OMAP2_MCSPI_CHCONF_DMAW         BIT(14)
  70 #define OMAP2_MCSPI_CHCONF_DMAR         BIT(15)
  71 #define OMAP2_MCSPI_CHCONF_DPE0         BIT(16)
  72 #define OMAP2_MCSPI_CHCONF_DPE1         BIT(17)
  73 #define OMAP2_MCSPI_CHCONF_IS           BIT(18)
  74 #define OMAP2_MCSPI_CHCONF_TURBO        BIT(19)
  75 #define OMAP2_MCSPI_CHCONF_FORCE        BIT(20)
  76 #define OMAP2_MCSPI_CHCONF_FFET         BIT(27)
  77 #define OMAP2_MCSPI_CHCONF_FFER         BIT(28)
  78 #define OMAP2_MCSPI_CHCONF_CLKG         BIT(29)
  79
  80 #define OMAP2_MCSPI_CHSTAT_RXS          BIT(0)
  81 #define OMAP2_MCSPI_CHSTAT_TXS          BIT(1)
  82 #define OMAP2_MCSPI_CHSTAT_EOT          BIT(2)
  83 #define OMAP2_MCSPI_CHSTAT_TXFFE        BIT(3)
  84
  85 #define OMAP2_MCSPI_CHCTRL_EN           BIT(0)
  86 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK  (0xff << 8)
  87
  88 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN   BIT(0)
  89
  90 /* We have 2 DMA channels per CS, one for RX and one for TX */
  91 struct omap2_mcspi_dma {
  92         struct dma_chan *dma_tx;
  93         struct dma_chan *dma_rx;
  94
  95         struct completion dma_tx_completion;
  96         struct completion dma_rx_completion;
  97
  98         char dma_rx_ch_name[14];
  99         char dma_tx_ch_name[14];
 100 };
 101
 102 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 103  * cache operations; better heuristics consider wordsize and bitrate.
 104  */
 105 #define DMA_MIN_BYTES                   160
 106
 107
 108 /*
 109  * Used for context save and restore, structure members to be updated whenever
 110  * corresponding registers are modified.
 111  */
 112 struct omap2_mcspi_regs {
 113         u32 modulctrl;
 114         u32 wakeupenable;
 115         struct list_head cs;
 116 };
 117
 118 struct omap2_mcspi {
 119         struct completion       txdone;
 120         struct spi_master       *master;
 121         /* Virtual base address of the controller */
 122         void __iomem            *base;
 123         unsigned long           phys;
 124         /* SPI1 has 4 channels, while SPI2 has 2 */
 125         struct omap2_mcspi_dma  *dma_channels;
 126         struct device           *dev;
 127         struct omap2_mcspi_regs ctx;
 128         int                     fifo_depth;
 129         bool                    slave_aborted;
 130         unsigned int            pin_dir:1;
 131         size_t                  max_xfer_len;
 132 };
 133
 134 struct omap2_mcspi_cs {
 135         void __iomem            *base;
 136         unsigned long           phys;
 137         int                     word_len;
 138         u16                     mode;
 139         struct list_head        node;
 140         /* Context save and restore shadow register */
 141         u32                     chconf0, chctrl0;
 142 };
 143
 144 static inline void mcspi_write_reg(struct spi_master *master,
 145                 int idx, u32 val)
 146 {
 147         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 148
 149         writel_relaxed(val, mcspi->base + idx);
 150 }
 151
 152 static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
 153 {
 154         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 155
 156         return readl_relaxed(mcspi->base + idx);
 157 }
 158
 159 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 160                 int idx, u32 val)
 161 {
 162         struct omap2_mcspi_cs   *cs = spi->controller_state;
 163
 164         writel_relaxed(val, cs->base +  idx);
 165 }
 166
 167 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 168 {
 169         struct omap2_mcspi_cs   *cs = spi->controller_state;
 170
 171         return readl_relaxed(cs->base + idx);
 172 }
 173
 174 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 175 {
 176         struct omap2_mcspi_cs *cs = spi->controller_state;
 177
 178         return cs->chconf0;
 179 }
 180
 181 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 182 {
 183         struct omap2_mcspi_cs *cs = spi->controller_state;
 184
 185         cs->chconf0 = val;
 186         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 187         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 188 }
 189
 190 static inline int mcspi_bytes_per_word(int word_len)
 191 {
 192         if (word_len <= 8)
 193                 return 1;
 194         else if (word_len <= 16)
 195                 return 2;
 196         else /* word_len <= 32 */
 197                 return 4;
 198 }
 199
 200 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 201                 int is_read, int enable)
 202 {
 203         u32 l, rw;
 204
 205         l = mcspi_cached_chconf0(spi);
 206
 207         if (is_read) /* 1 is read, 0 write */
 208                 rw = OMAP2_MCSPI_CHCONF_DMAR;
 209         else
 210                 rw = OMAP2_MCSPI_CHCONF_DMAW;
 211
 212         if (enable)
 213                 l |= rw;
 214         else
 215                 l &= ~rw;
 216
 217         mcspi_write_chconf0(spi, l);
 218 }
 219
 220 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 221 {
 222         struct omap2_mcspi_cs *cs = spi->controller_state;
 223         u32 l;
 224
 225         l = cs->chctrl0;
 226         if (enable)
 227                 l |= OMAP2_MCSPI_CHCTRL_EN;
 228         else
 229                 l &= ~OMAP2_MCSPI_CHCTRL_EN;
 230         cs->chctrl0 = l;
 231         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 232         /* Flash post-writes */
 233         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 234 }
 235
 236 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 237 {
 238         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 239         u32 l;
 240
 241         /* The controller handles the inverted chip selects
 242          * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 243          * the inversion from the core spi_set_cs function.
 244          */
 245         if (spi->mode & SPI_CS_HIGH)
 246                 enable = !enable;
 247
 248         if (spi->controller_state) {
 249                 int err = pm_runtime_get_sync(mcspi->dev);
 250                 if (err < 0) {
 251                         pm_runtime_put_noidle(mcspi->dev);
 252                         dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 253                         return;
 254                 }
 255
 256                 l = mcspi_cached_chconf0(spi);
 257
 258                 if (enable)
 259                         l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 260                 else
 261                         l |= OMAP2_MCSPI_CHCONF_FORCE;
 262
 263                 mcspi_write_chconf0(spi, l);
 264
 265                 pm_runtime_mark_last_busy(mcspi->dev);
 266                 pm_runtime_put_autosuspend(mcspi->dev);
 267         }
 268 }
 269
 270 static void omap2_mcspi_set_mode(struct spi_master *master)
 271 {
 272         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
 273         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
 274         u32 l;
 275
 276         /*
 277          * Choose master or slave mode
 278          */
 279         l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
 280         l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
 281         if (spi_controller_is_slave(master)) {
 282                 l |= (OMAP2_MCSPI_MODULCTRL_MS);
 283         } else {
 284                 l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
 285                 l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 286         }
 287         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 288
 289         ctx->modulctrl = l;
 290 }
 291
 292 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 293                                 struct spi_transfer *t, int enable)
 294 {
 295         struct spi_master *master = spi->master;
 296         struct omap2_mcspi_cs *cs = spi->controller_state;
 297         struct omap2_mcspi *mcspi;
 298         unsigned int wcnt;
 299         int max_fifo_depth, bytes_per_word;
 300         u32 chconf, xferlevel;
 301
 302         mcspi = spi_master_get_devdata(master);
 303
 304         chconf = mcspi_cached_chconf0(spi);
 305         if (enable) {
 306                 bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 307                 if (t->len % bytes_per_word != 0)
 308                         goto disable_fifo;
 309
 310                 if (t->rx_buf != NULL && t->tx_buf != NULL)
 311                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 312                 else
 313                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 314
 315                 wcnt = t->len / bytes_per_word;
 316                 if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 317                         goto disable_fifo;
 318
 319                 xferlevel = wcnt << 16;
 320                 if (t->rx_buf != NULL) {
 321                         chconf |= OMAP2_MCSPI_CHCONF_FFER;
 322                         xferlevel |= (bytes_per_word - 1) << 8;
 323                 }
 324
 325                 if (t->tx_buf != NULL) {
 326                         chconf |= OMAP2_MCSPI_CHCONF_FFET;
 327                         xferlevel |= bytes_per_word - 1;
 328                 }
 329
 330                 mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 331                 mcspi_write_chconf0(spi, chconf);
 332                 mcspi->fifo_depth = max_fifo_depth;
 333
 334                 return;
 335         }
 336
 337 disable_fifo:
 338         if (t->rx_buf != NULL)
 339                 chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 340
 341         if (t->tx_buf != NULL)
 342                 chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 343
 344         mcspi_write_chconf0(spi, chconf);
 345         mcspi->fifo_depth = 0;
 346 }
 347
 348 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 349 {
 350         unsigned long timeout;
 351
 352         timeout = jiffies + msecs_to_jiffies(1000);
 353         while (!(readl_relaxed(reg) & bit)) {
 354                 if (time_after(jiffies, timeout)) {
 355                         if (!(readl_relaxed(reg) & bit))
 356                                 return -ETIMEDOUT;
 357                         else
 358                                 return 0;
 359                 }
 360                 cpu_relax();
 361         }
 362         return 0;
 363 }
 364
 365 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
 366                                      struct completion *x)
 367 {
 368         if (spi_controller_is_slave(mcspi->master)) {
 369                 if (wait_for_completion_interruptible(x) ||
 370                     mcspi->slave_aborted)
 371                         return -EINTR;
 372         } else {
 373                 wait_for_completion(x);
 374         }
 375
 376         return 0;
 377 }
 378
 379 static void omap2_mcspi_rx_callback(void *data)
 380 {
 381         struct spi_device *spi = data;
 382         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 383         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 384
 385         /* We must disable the DMA RX request */
 386         omap2_mcspi_set_dma_req(spi, 1, 0);
 387
 388         complete(&mcspi_dma->dma_rx_completion);
 389 }
 390
 391 static void omap2_mcspi_tx_callback(void *data)
 392 {
 393         struct spi_device *spi = data;
 394         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 395         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 396
 397         /* We must disable the DMA TX request */
 398         omap2_mcspi_set_dma_req(spi, 0, 0);
 399
 400         complete(&mcspi_dma->dma_tx_completion);
 401 }
 402
 403 static void omap2_mcspi_tx_dma(struct spi_device *spi,
 404                                 struct spi_transfer *xfer,
 405                                 struct dma_slave_config cfg)
 406 {
 407         struct omap2_mcspi      *mcspi;
 408         struct omap2_mcspi_dma  *mcspi_dma;
 409         struct dma_async_tx_descriptor *tx;
 410
 411         mcspi = spi_master_get_devdata(spi->master);
 412         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 413
 414         dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 415
 416         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 417                                      xfer->tx_sg.nents,
 418                                      DMA_MEM_TO_DEV,
 419                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 420         if (tx) {
 421                 tx->callback = omap2_mcspi_tx_callback;
 422                 tx->callback_param = spi;
 423                 dmaengine_submit(tx);
 424         } else {
 425                 /* FIXME: fall back to PIO? */
 426         }
 427         dma_async_issue_pending(mcspi_dma->dma_tx);
 428         omap2_mcspi_set_dma_req(spi, 0, 1);
 429 }
 430
 431 static unsigned
 432 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 433                                 struct dma_slave_config cfg,
 434                                 unsigned es)
 435 {
 436         struct omap2_mcspi      *mcspi;
 437         struct omap2_mcspi_dma  *mcspi_dma;
 438         unsigned int            count, transfer_reduction = 0;
 439         struct scatterlist      *sg_out[2];
 440         int                     nb_sizes = 0, out_mapped_nents[2], ret, x;
 441         size_t                  sizes[2];
 442         u32                     l;
 443         int                     elements = 0;
 444         int                     word_len, element_count;
 445         struct omap2_mcspi_cs   *cs = spi->controller_state;
 446         void __iomem            *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 447         struct dma_async_tx_descriptor *tx;
 448
 449         mcspi = spi_master_get_devdata(spi->master);
 450         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 451         count = xfer->len;
 452
 453         /*
 454          *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 455          *  it mentions reducing DMA transfer length by one element in master
 456          *  normal mode.
 457          */
 458         if (mcspi->fifo_depth == 0)
 459                 transfer_reduction = es;
 460
 461         word_len = cs->word_len;
 462         l = mcspi_cached_chconf0(spi);
 463
 464         if (word_len <= 8)
 465                 element_count = count;
 466         else if (word_len <= 16)
 467                 element_count = count >> 1;
 468         else /* word_len <= 32 */
 469                 element_count = count >> 2;
 470
 471
 472         dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 473
 474         /*
 475          *  Reduce DMA transfer length by one more if McSPI is
 476          *  configured in turbo mode.
 477          */
 478         if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 479                 transfer_reduction += es;
 480
 481         if (transfer_reduction) {
 482                 /* Split sgl into two. The second sgl won't be used. */
 483                 sizes[0] = count - transfer_reduction;
 484                 sizes[1] = transfer_reduction;
 485                 nb_sizes = 2;
 486         } else {
 487                 /*
 488                  * Don't bother splitting the sgl. This essentially
 489                  * clones the original sgl.
 490                  */
 491                 sizes[0] = count;
 492                 nb_sizes = 1;
 493         }
 494
 495         ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
 496                        sizes, sg_out, out_mapped_nents, GFP_KERNEL);
 497
 498         if (ret < 0) {
 499                 dev_err(&spi->dev, "sg_split failed\n");
 500                 return 0;
 501         }
 502
 503         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
 504                                      out_mapped_nents[0], DMA_DEV_TO_MEM,
 505                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 506         if (tx) {
 507                 tx->callback = omap2_mcspi_rx_callback;
 508                 tx->callback_param = spi;
 509                 dmaengine_submit(tx);
 510         } else {
 511                 /* FIXME: fall back to PIO? */
 512         }
 513
 514         dma_async_issue_pending(mcspi_dma->dma_rx);
 515         omap2_mcspi_set_dma_req(spi, 1, 1);
 516
 517         ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
 518         if (ret || mcspi->slave_aborted) {
 519                 dmaengine_terminate_sync(mcspi_dma->dma_rx);
 520                 omap2_mcspi_set_dma_req(spi, 1, 0);
 521                 return 0;
 522         }
 523
 524         for (x = 0; x < nb_sizes; x++)
 525                 kfree(sg_out[x]);
 526
 527         if (mcspi->fifo_depth > 0)
 528                 return count;
 529
 530         /*
 531          *  Due to the DMA transfer length reduction the missing bytes must
 532          *  be read manually to receive all of the expected data.
 533          */
 534         omap2_mcspi_set_enable(spi, 0);
 535
 536         elements = element_count - 1;
 537
 538         if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 539                 elements--;
 540
 541                 if (!mcspi_wait_for_reg_bit(chstat_reg,
 542                                             OMAP2_MCSPI_CHSTAT_RXS)) {
 543                         u32 w;
 544
 545                         w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 546                         if (word_len <= 8)
 547                                 ((u8 *)xfer->rx_buf)[elements++] = w;
 548                         else if (word_len <= 16)
 549                                 ((u16 *)xfer->rx_buf)[elements++] = w;
 550                         else /* word_len <= 32 */
 551                                 ((u32 *)xfer->rx_buf)[elements++] = w;
 552                 } else {
 553                         int bytes_per_word = mcspi_bytes_per_word(word_len);
 554                         dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 555                         count -= (bytes_per_word << 1);
 556                         omap2_mcspi_set_enable(spi, 1);
 557                         return count;
 558                 }
 559         }
 560         if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
 561                 u32 w;
 562
 563                 w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 564                 if (word_len <= 8)
 565                         ((u8 *)xfer->rx_buf)[elements] = w;
 566                 else if (word_len <= 16)
 567                         ((u16 *)xfer->rx_buf)[elements] = w;
 568                 else /* word_len <= 32 */
 569                         ((u32 *)xfer->rx_buf)[elements] = w;
 570         } else {
 571                 dev_err(&spi->dev, "DMA RX last word empty\n");
 572                 count -= mcspi_bytes_per_word(word_len);
 573         }
 574         omap2_mcspi_set_enable(spi, 1);
 575         return count;
 576 }
 577
 578 static unsigned
 579 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 580 {
 581         struct omap2_mcspi      *mcspi;
 582         struct omap2_mcspi_cs   *cs = spi->controller_state;
 583         struct omap2_mcspi_dma  *mcspi_dma;
 584         unsigned int            count;
 585         u8                      *rx;
 586         const u8                *tx;
 587         struct dma_slave_config cfg;
 588         enum dma_slave_buswidth width;
 589         unsigned es;
 590         void __iomem            *chstat_reg;
 591         void __iomem            *irqstat_reg;
 592         int                     wait_res;
 593
 594         mcspi = spi_master_get_devdata(spi->master);
 595         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 596
 597         if (cs->word_len <= 8) {
 598                 width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 599                 es = 1;
 600         } else if (cs->word_len <= 16) {
 601                 width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 602                 es = 2;
 603         } else {
 604                 width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 605                 es = 4;
 606         }
 607
 608         count = xfer->len;
 609
 610         memset(&cfg, 0, sizeof(cfg));
 611         cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 612         cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 613         cfg.src_addr_width = width;
 614         cfg.dst_addr_width = width;
 615         cfg.src_maxburst = 1;
 616         cfg.dst_maxburst = 1;
 617
 618         rx = xfer->rx_buf;
 619         tx = xfer->tx_buf;
 620
 621         mcspi->slave_aborted = false;
 622         reinit_completion(&mcspi_dma->dma_tx_completion);
 623         reinit_completion(&mcspi_dma->dma_rx_completion);
 624         reinit_completion(&mcspi->txdone);
 625         if (tx) {
 626                 /* Enable EOW IRQ to know end of tx in slave mode */
 627                 if (spi_controller_is_slave(spi->master))
 628                         mcspi_write_reg(spi->master,
 629                                         OMAP2_MCSPI_IRQENABLE,
 630                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 631                 omap2_mcspi_tx_dma(spi, xfer, cfg);
 632         }
 633
 634         if (rx != NULL)
 635                 count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 636
 637         if (tx != NULL) {
 638                 int ret;
 639
 640                 ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
 641                 if (ret || mcspi->slave_aborted) {
 642                         dmaengine_terminate_sync(mcspi_dma->dma_tx);
 643                         omap2_mcspi_set_dma_req(spi, 0, 0);
 644                         return 0;
 645                 }
 646
 647                 if (spi_controller_is_slave(mcspi->master)) {
 648                         ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
 649                         if (ret || mcspi->slave_aborted)
 650                                 return 0;
 651                 }
 652
 653                 if (mcspi->fifo_depth > 0) {
 654                         irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 655
 656                         if (mcspi_wait_for_reg_bit(irqstat_reg,
 657                                                 OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 658                                 dev_err(&spi->dev, "EOW timed out\n");
 659
 660                         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
 661                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 662                 }
 663
 664                 /* for TX_ONLY mode, be sure all words have shifted out */
 665                 if (rx == NULL) {
 666                         chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 667                         if (mcspi->fifo_depth > 0) {
 668                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 669                                                 OMAP2_MCSPI_CHSTAT_TXFFE);
 670                                 if (wait_res < 0)
 671                                         dev_err(&spi->dev, "TXFFE timed out\n");
 672                         } else {
 673                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 674                                                 OMAP2_MCSPI_CHSTAT_TXS);
 675                                 if (wait_res < 0)
 676                                         dev_err(&spi->dev, "TXS timed out\n");
 677                         }
 678                         if (wait_res >= 0 &&
 679                                 (mcspi_wait_for_reg_bit(chstat_reg,
 680                                         OMAP2_MCSPI_CHSTAT_EOT) < 0))
 681                                 dev_err(&spi->dev, "EOT timed out\n");
 682                 }
 683         }
 684         return count;
 685 }
 686
 687 static unsigned
 688 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 689 {
 690         struct omap2_mcspi_cs   *cs = spi->controller_state;
 691         unsigned int            count, c;
 692         u32                     l;
 693         void __iomem            *base = cs->base;
 694         void __iomem            *tx_reg;
 695         void __iomem            *rx_reg;
 696         void __iomem            *chstat_reg;
 697         int                     word_len;
 698
 699         count = xfer->len;
 700         c = count;
 701         word_len = cs->word_len;
 702
 703         l = mcspi_cached_chconf0(spi);
 704
 705         /* We store the pre-calculated register addresses on stack to speed
 706          * up the transfer loop. */
 707         tx_reg          = base + OMAP2_MCSPI_TX0;
 708         rx_reg          = base + OMAP2_MCSPI_RX0;
 709         chstat_reg      = base + OMAP2_MCSPI_CHSTAT0;
 710
 711         if (c < (word_len>>3))
 712                 return 0;
 713
 714         if (word_len <= 8) {
 715                 u8              *rx;
 716                 const u8        *tx;
 717
 718                 rx = xfer->rx_buf;
 719                 tx = xfer->tx_buf;
 720
 721                 do {
 722                         c -= 1;
 723                         if (tx != NULL) {
 724                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 725                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 726                                         dev_err(&spi->dev, "TXS timed out\n");
 727                                         goto out;
 728                                 }
 729                                 dev_vdbg(&spi->dev, "write-%d %02x\n",
 730                                                 word_len, *tx);
 731                                 writel_relaxed(*tx++, tx_reg);
 732                         }
 733                         if (rx != NULL) {
 734                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 735                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 736                                         dev_err(&spi->dev, "RXS timed out\n");
 737                                         goto out;
 738                                 }
 739
 740                                 if (c == 1 && tx == NULL &&
 741                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 742                                         omap2_mcspi_set_enable(spi, 0);
 743                                         *rx++ = readl_relaxed(rx_reg);
 744                                         dev_vdbg(&spi->dev, "read-%d %02x\n",
 745                                                     word_len, *(rx - 1));
 746                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 747                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 748                                                 dev_err(&spi->dev,
 749                                                         "RXS timed out\n");
 750                                                 goto out;
 751                                         }
 752                                         c = 0;
 753                                 } else if (c == 0 && tx == NULL) {
 754                                         omap2_mcspi_set_enable(spi, 0);
 755                                 }
 756
 757                                 *rx++ = readl_relaxed(rx_reg);
 758                                 dev_vdbg(&spi->dev, "read-%d %02x\n",
 759                                                 word_len, *(rx - 1));
 760                         }
 761                 } while (c);
 762         } else if (word_len <= 16) {
 763                 u16             *rx;
 764                 const u16       *tx;
 765
 766                 rx = xfer->rx_buf;
 767                 tx = xfer->tx_buf;
 768                 do {
 769                         c -= 2;
 770                         if (tx != NULL) {
 771                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 772                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 773                                         dev_err(&spi->dev, "TXS timed out\n");
 774                                         goto out;
 775                                 }
 776                                 dev_vdbg(&spi->dev, "write-%d %04x\n",
 777                                                 word_len, *tx);
 778                                 writel_relaxed(*tx++, tx_reg);
 779                         }
 780                         if (rx != NULL) {
 781                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 782                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 783                                         dev_err(&spi->dev, "RXS timed out\n");
 784                                         goto out;
 785                                 }
 786
 787                                 if (c == 2 && tx == NULL &&
 788                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 789                                         omap2_mcspi_set_enable(spi, 0);
 790                                         *rx++ = readl_relaxed(rx_reg);
 791                                         dev_vdbg(&spi->dev, "read-%d %04x\n",
 792                                                     word_len, *(rx - 1));
 793                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 794                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 795                                                 dev_err(&spi->dev,
 796                                                         "RXS timed out\n");
 797                                                 goto out;
 798                                         }
 799                                         c = 0;
 800                                 } else if (c == 0 && tx == NULL) {
 801                                         omap2_mcspi_set_enable(spi, 0);
 802                                 }
 803
 804                                 *rx++ = readl_relaxed(rx_reg);
 805                                 dev_vdbg(&spi->dev, "read-%d %04x\n",
 806                                                 word_len, *(rx - 1));
 807                         }
 808                 } while (c >= 2);
 809         } else if (word_len <= 32) {
 810                 u32             *rx;
 811                 const u32       *tx;
 812
 813                 rx = xfer->rx_buf;
 814                 tx = xfer->tx_buf;
 815                 do {
 816                         c -= 4;
 817                         if (tx != NULL) {
 818                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 819                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 820                                         dev_err(&spi->dev, "TXS timed out\n");
 821                                         goto out;
 822                                 }
 823                                 dev_vdbg(&spi->dev, "write-%d %08x\n",
 824                                                 word_len, *tx);
 825                                 writel_relaxed(*tx++, tx_reg);
 826                         }
 827                         if (rx != NULL) {
 828                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 829                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 830                                         dev_err(&spi->dev, "RXS timed out\n");
 831                                         goto out;
 832                                 }
 833
 834                                 if (c == 4 && tx == NULL &&
 835                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 836                                         omap2_mcspi_set_enable(spi, 0);
 837                                         *rx++ = readl_relaxed(rx_reg);
 838                                         dev_vdbg(&spi->dev, "read-%d %08x\n",
 839                                                     word_len, *(rx - 1));
 840                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 841                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 842                                                 dev_err(&spi->dev,
 843                                                         "RXS timed out\n");
 844                                                 goto out;
 845                                         }
 846                                         c = 0;
 847                                 } else if (c == 0 && tx == NULL) {
 848                                         omap2_mcspi_set_enable(spi, 0);
 849                                 }
 850
 851                                 *rx++ = readl_relaxed(rx_reg);
 852                                 dev_vdbg(&spi->dev, "read-%d %08x\n",
 853                                                 word_len, *(rx - 1));
 854                         }
 855                 } while (c >= 4);
 856         }
 857
 858         /* for TX_ONLY mode, be sure all words have shifted out */
 859         if (xfer->rx_buf == NULL) {
 860                 if (mcspi_wait_for_reg_bit(chstat_reg,
 861                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 862                         dev_err(&spi->dev, "TXS timed out\n");
 863                 } else if (mcspi_wait_for_reg_bit(chstat_reg,
 864                                 OMAP2_MCSPI_CHSTAT_EOT) < 0)
 865                         dev_err(&spi->dev, "EOT timed out\n");
 866
 867                 /* disable chan to purge rx datas received in TX_ONLY transfer,
 868                  * otherwise these rx datas will affect the direct following
 869                  * RX_ONLY transfer.
 870                  */
 871                 omap2_mcspi_set_enable(spi, 0);
 872         }
 873 out:
 874         omap2_mcspi_set_enable(spi, 1);
 875         return count - c;
 876 }
 877
 878 static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
 879 {
 880         u32 div;
 881
 882         for (div = 0; div < 15; div++)
 883                 if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
 884                         return div;
 885
 886         return 15;
 887 }
 888
 889 /* called only when no transfer is active to this device */
 890 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 891                 struct spi_transfer *t)
 892 {
 893         struct omap2_mcspi_cs *cs = spi->controller_state;
 894         struct omap2_mcspi *mcspi;
 895         u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 896         u8 word_len = spi->bits_per_word;
 897         u32 speed_hz = spi->max_speed_hz;
 898
 899         mcspi = spi_master_get_devdata(spi->master);
 900
 901         if (t != NULL && t->bits_per_word)
 902                 word_len = t->bits_per_word;
 903
 904         cs->word_len = word_len;
 905
 906         if (t && t->speed_hz)
 907                 speed_hz = t->speed_hz;
 908
 909         speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
 910         if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
 911                 clkd = omap2_mcspi_calc_divisor(speed_hz);
 912                 speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
 913                 clkg = 0;
 914         } else {
 915                 div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
 916                 speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
 917                 clkd = (div - 1) & 0xf;
 918                 extclk = (div - 1) >> 4;
 919                 clkg = OMAP2_MCSPI_CHCONF_CLKG;
 920         }
 921
 922         l = mcspi_cached_chconf0(spi);
 923
 924         /* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
 925          * REVISIT: this controller could support SPI_3WIRE mode.
 926          */
 927         if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 928                 l &= ~OMAP2_MCSPI_CHCONF_IS;
 929                 l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 930                 l |= OMAP2_MCSPI_CHCONF_DPE0;
 931         } else {
 932                 l |= OMAP2_MCSPI_CHCONF_IS;
 933                 l |= OMAP2_MCSPI_CHCONF_DPE1;
 934                 l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 935         }
 936
 937         /* wordlength */
 938         l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 939         l |= (word_len - 1) << 7;
 940
 941         /* set chipselect polarity; manage with FORCE */
 942         if (!(spi->mode & SPI_CS_HIGH))
 943                 l |= OMAP2_MCSPI_CHCONF_EPOL;   /* active-low; normal */
 944         else
 945                 l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 946
 947         /* set clock divisor */
 948         l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 949         l |= clkd << 2;
 950
 951         /* set clock granularity */
 952         l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 953         l |= clkg;
 954         if (clkg) {
 955                 cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 956                 cs->chctrl0 |= extclk << 8;
 957                 mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 958         }
 959
 960         /* set SPI mode 0..3 */
 961         if (spi->mode & SPI_CPOL)
 962                 l |= OMAP2_MCSPI_CHCONF_POL;
 963         else
 964                 l &= ~OMAP2_MCSPI_CHCONF_POL;
 965         if (spi->mode & SPI_CPHA)
 966                 l |= OMAP2_MCSPI_CHCONF_PHA;
 967         else
 968                 l &= ~OMAP2_MCSPI_CHCONF_PHA;
 969
 970         mcspi_write_chconf0(spi, l);
 971
 972         cs->mode = spi->mode;
 973
 974         dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 975                         speed_hz,
 976                         (spi->mode & SPI_CPHA) ? "trailing" : "leading",
 977                         (spi->mode & SPI_CPOL) ? "inverted" : "normal");
 978
 979         return 0;
 980 }
 981
 982 /*
 983  * Note that we currently allow DMA only if we get a channel
 984  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
 985  */
 986 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
 987                                    struct omap2_mcspi_dma *mcspi_dma)
 988 {
 989         int ret = 0;
 990
 991         mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
 992                                              mcspi_dma->dma_rx_ch_name);
 993         if (IS_ERR(mcspi_dma->dma_rx)) {
 994                 ret = PTR_ERR(mcspi_dma->dma_rx);
 995                 mcspi_dma->dma_rx = NULL;
 996                 goto no_dma;
 997         }
 998
 999         mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
1000                                              mcspi_dma->dma_tx_ch_name);
1001         if (IS_ERR(mcspi_dma->dma_tx)) {
1002                 ret = PTR_ERR(mcspi_dma->dma_tx);
1003                 mcspi_dma->dma_tx = NULL;
1004                 dma_release_channel(mcspi_dma->dma_rx);
1005                 mcspi_dma->dma_rx = NULL;
1006         }
1007
1008         init_completion(&mcspi_dma->dma_rx_completion);
1009         init_completion(&mcspi_dma->dma_tx_completion);
1010
1011 no_dma:
1012         return ret;
1013 }
1014
1015 static void omap2_mcspi_release_dma(struct spi_master *master)
1016 {
1017         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1018         struct omap2_mcspi_dma  *mcspi_dma;
1019         int i;
1020
1021         for (i = 0; i < master->num_chipselect; i++) {
1022                 mcspi_dma = &mcspi->dma_channels[i];
1023
1024                 if (mcspi_dma->dma_rx) {
1025                         dma_release_channel(mcspi_dma->dma_rx);
1026                         mcspi_dma->dma_rx = NULL;
1027                 }
1028                 if (mcspi_dma->dma_tx) {
1029                         dma_release_channel(mcspi_dma->dma_tx);
1030                         mcspi_dma->dma_tx = NULL;
1031                 }
1032         }
1033 }
1034
1035 static void omap2_mcspi_cleanup(struct spi_device *spi)
1036 {
1037         struct omap2_mcspi_cs   *cs;
1038
1039         if (spi->controller_state) {
1040                 /* Unlink controller state from context save list */
1041                 cs = spi->controller_state;
1042                 list_del(&cs->node);
1043
1044                 kfree(cs);
1045         }
1046 }
1047
1048 static int omap2_mcspi_setup(struct spi_device *spi)
1049 {
1050         bool                    initial_setup = false;
1051         int                     ret;
1052         struct omap2_mcspi      *mcspi = spi_master_get_devdata(spi->master);
1053         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1054         struct omap2_mcspi_cs   *cs = spi->controller_state;
1055
1056         if (!cs) {
1057                 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
1058                 if (!cs)
1059                         return -ENOMEM;
1060                 cs->base = mcspi->base + spi->chip_select * 0x14;
1061                 cs->phys = mcspi->phys + spi->chip_select * 0x14;
1062                 cs->mode = 0;
1063                 cs->chconf0 = 0;
1064                 cs->chctrl0 = 0;
1065                 spi->controller_state = cs;
1066                 /* Link this to context save list */
1067                 list_add_tail(&cs->node, &ctx->cs);
1068                 initial_setup = true;
1069         }
1070
1071         ret = pm_runtime_get_sync(mcspi->dev);
1072         if (ret < 0) {
1073                 pm_runtime_put_noidle(mcspi->dev);
1074                 if (initial_setup)
1075                         omap2_mcspi_cleanup(spi);
1076
1077                 return ret;
1078         }
1079
1080         ret = omap2_mcspi_setup_transfer(spi, NULL);
1081         if (ret && initial_setup)
1082                 omap2_mcspi_cleanup(spi);
1083
1084         pm_runtime_mark_last_busy(mcspi->dev);
1085         pm_runtime_put_autosuspend(mcspi->dev);
1086
1087         return ret;
1088 }
1089
1090 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1091 {
1092         struct omap2_mcspi *mcspi = data;
1093         u32 irqstat;
1094
1095         irqstat = mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
1096         if (!irqstat)
1097                 return IRQ_NONE;
1098
1099         /* Disable IRQ and wakeup slave xfer task */
1100         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
1101         if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1102                 complete(&mcspi->txdone);
1103
1104         return IRQ_HANDLED;
1105 }
1106
1107 static int omap2_mcspi_slave_abort(struct spi_master *master)
1108 {
1109         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1110         struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1111
1112         mcspi->slave_aborted = true;
1113         complete(&mcspi_dma->dma_rx_completion);
1114         complete(&mcspi_dma->dma_tx_completion);
1115         complete(&mcspi->txdone);
1116
1117         return 0;
1118 }
1119
1120 static int omap2_mcspi_transfer_one(struct spi_master *master,
1121                                     struct spi_device *spi,
1122                                     struct spi_transfer *t)
1123 {
1124
1125         /* We only enable one channel at a time -- the one whose message is
1126          * -- although this controller would gladly
1127          * arbitrate among multiple channels.  This corresponds to "single
1128          * channel" master mode.  As a side effect, we need to manage the
1129          * chipselect with the FORCE bit ... CS != channel enable.
1130          */
1131
1132         struct omap2_mcspi              *mcspi;
1133         struct omap2_mcspi_dma          *mcspi_dma;
1134         struct omap2_mcspi_cs           *cs;
1135         struct omap2_mcspi_device_config *cd;
1136         int                             par_override = 0;
1137         int                             status = 0;
1138         u32                             chconf;
1139
1140         mcspi = spi_master_get_devdata(master);
1141         mcspi_dma = mcspi->dma_channels + spi->chip_select;
1142         cs = spi->controller_state;
1143         cd = spi->controller_data;
1144
1145         /*
1146          * The slave driver could have changed spi->mode in which case
1147          * it will be different from cs->mode (the current hardware setup).
1148          * If so, set par_override (even though its not a parity issue) so
1149          * omap2_mcspi_setup_transfer will be called to configure the hardware
1150          * with the correct mode on the first iteration of the loop below.
1151          */
1152         if (spi->mode != cs->mode)
1153                 par_override = 1;
1154
1155         omap2_mcspi_set_enable(spi, 0);
1156
1157         if (spi->cs_gpiod)
1158                 omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1159
1160         if (par_override ||
1161             (t->speed_hz != spi->max_speed_hz) ||
1162             (t->bits_per_word != spi->bits_per_word)) {
1163                 par_override = 1;
1164                 status = omap2_mcspi_setup_transfer(spi, t);
1165                 if (status < 0)
1166                         goto out;
1167                 if (t->speed_hz == spi->max_speed_hz &&
1168                     t->bits_per_word == spi->bits_per_word)
1169                         par_override = 0;
1170         }
1171         if (cd && cd->cs_per_word) {
1172                 chconf = mcspi->ctx.modulctrl;
1173                 chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1174                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1175                 mcspi->ctx.modulctrl =
1176                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1177         }
1178
1179         chconf = mcspi_cached_chconf0(spi);
1180         chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1181         chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1182
1183         if (t->tx_buf == NULL)
1184                 chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1185         else if (t->rx_buf == NULL)
1186                 chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1187
1188         if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1189                 /* Turbo mode is for more than one word */
1190                 if (t->len > ((cs->word_len + 7) >> 3))
1191                         chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1192         }
1193
1194         mcspi_write_chconf0(spi, chconf);
1195
1196         if (t->len) {
1197                 unsigned        count;
1198
1199                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1200                     master->cur_msg_mapped &&
1201                     master->can_dma(master, spi, t))
1202                         omap2_mcspi_set_fifo(spi, t, 1);
1203
1204                 omap2_mcspi_set_enable(spi, 1);
1205
1206                 /* RX_ONLY mode needs dummy data in TX reg */
1207                 if (t->tx_buf == NULL)
1208                         writel_relaxed(0, cs->base
1209                                         + OMAP2_MCSPI_TX0);
1210
1211                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1212                     master->cur_msg_mapped &&
1213                     master->can_dma(master, spi, t))
1214                         count = omap2_mcspi_txrx_dma(spi, t);
1215                 else
1216                         count = omap2_mcspi_txrx_pio(spi, t);
1217
1218                 if (count != t->len) {
1219                         status = -EIO;
1220                         goto out;
1221                 }
1222         }
1223
1224         omap2_mcspi_set_enable(spi, 0);
1225
1226         if (mcspi->fifo_depth > 0)
1227                 omap2_mcspi_set_fifo(spi, t, 0);
1228
1229 out:
1230         /* Restore defaults if they were overriden */
1231         if (par_override) {
1232                 par_override = 0;
1233                 status = omap2_mcspi_setup_transfer(spi, NULL);
1234         }
1235
1236         if (cd && cd->cs_per_word) {
1237                 chconf = mcspi->ctx.modulctrl;
1238                 chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1239                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1240                 mcspi->ctx.modulctrl =
1241                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1242         }
1243
1244         omap2_mcspi_set_enable(spi, 0);
1245
1246         if (spi->cs_gpiod)
1247                 omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1248
1249         if (mcspi->fifo_depth > 0 && t)
1250                 omap2_mcspi_set_fifo(spi, t, 0);
1251
1252         return status;
1253 }
1254
1255 static int omap2_mcspi_prepare_message(struct spi_master *master,
1256                                        struct spi_message *msg)
1257 {
1258         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
1259         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1260         struct omap2_mcspi_cs   *cs;
1261
1262         /* Only a single channel can have the FORCE bit enabled
1263          * in its chconf0 register.
1264          * Scan all channels and disable them except the current one.
1265          * A FORCE can remain from a last transfer having cs_change enabled
1266          */
1267         list_for_each_entry(cs, &ctx->cs, node) {
1268                 if (msg->spi->controller_state == cs)
1269                         continue;
1270
1271                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1272                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1273                         writel_relaxed(cs->chconf0,
1274                                         cs->base + OMAP2_MCSPI_CHCONF0);
1275                         readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1276                 }
1277         }
1278
1279         return 0;
1280 }
1281
1282 static bool omap2_mcspi_can_dma(struct spi_master *master,
1283                                 struct spi_device *spi,
1284                                 struct spi_transfer *xfer)
1285 {
1286         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1287         struct omap2_mcspi_dma *mcspi_dma =
1288                 &mcspi->dma_channels[spi->chip_select];
1289
1290         if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1291                 return false;
1292
1293         if (spi_controller_is_slave(master))
1294                 return true;
1295
1296         master->dma_rx = mcspi_dma->dma_rx;
1297         master->dma_tx = mcspi_dma->dma_tx;
1298
1299         return (xfer->len >= DMA_MIN_BYTES);
1300 }
1301
1302 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1303 {
1304         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1305         struct omap2_mcspi_dma *mcspi_dma =
1306                 &mcspi->dma_channels[spi->chip_select];
1307
1308         if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1309                 return mcspi->max_xfer_len;
1310
1311         return SIZE_MAX;
1312 }
1313
1314 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1315 {
1316         struct spi_master       *master = mcspi->master;
1317         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1318         int                     ret = 0;
1319
1320         ret = pm_runtime_get_sync(mcspi->dev);
1321         if (ret < 0) {
1322                 pm_runtime_put_noidle(mcspi->dev);
1323
1324                 return ret;
1325         }
1326
1327         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1328                         OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1329         ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1330
1331         omap2_mcspi_set_mode(master);
1332         pm_runtime_mark_last_busy(mcspi->dev);
1333         pm_runtime_put_autosuspend(mcspi->dev);
1334         return 0;
1335 }
1336
1337 static int omap_mcspi_runtime_suspend(struct device *dev)
1338 {
1339         int error;
1340
1341         error = pinctrl_pm_select_idle_state(dev);
1342         if (error)
1343                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1344
1345         return 0;
1346 }
1347
1348 /*
1349  * When SPI wake up from off-mode, CS is in activate state. If it was in
1350  * inactive state when driver was suspend, then force it to inactive state at
1351  * wake up.
1352  */
1353 static int omap_mcspi_runtime_resume(struct device *dev)
1354 {
1355         struct spi_master *master = dev_get_drvdata(dev);
1356         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1357         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1358         struct omap2_mcspi_cs *cs;
1359         int error;
1360
1361         error = pinctrl_pm_select_default_state(dev);
1362         if (error)
1363                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1364
1365         /* McSPI: context restore */
1366         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1367         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1368
1369         list_for_each_entry(cs, &ctx->cs, node) {
1370                 /*
1371                  * We need to toggle CS state for OMAP take this
1372                  * change in account.
1373                  */
1374                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1375                         cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1376                         writel_relaxed(cs->chconf0,
1377                                        cs->base + OMAP2_MCSPI_CHCONF0);
1378                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1379                         writel_relaxed(cs->chconf0,
1380                                        cs->base + OMAP2_MCSPI_CHCONF0);
1381                 } else {
1382                         writel_relaxed(cs->chconf0,
1383                                        cs->base + OMAP2_MCSPI_CHCONF0);
1384                 }
1385         }
1386
1387         return 0;
1388 }
1389
1390 static struct omap2_mcspi_platform_config omap2_pdata = {
1391         .regs_offset = 0,
1392 };
1393
1394 static struct omap2_mcspi_platform_config omap4_pdata = {
1395         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1396 };
1397
1398 static struct omap2_mcspi_platform_config am654_pdata = {
1399         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1400         .max_xfer_len = SZ_4K - 1,
1401 };
1402
1403 static const struct of_device_id omap_mcspi_of_match[] = {
1404         {
1405                 .compatible = "ti,omap2-mcspi",
1406                 .data = &omap2_pdata,
1407         },
1408         {
1409                 .compatible = "ti,omap4-mcspi",
1410                 .data = &omap4_pdata,
1411         },
1412         {
1413                 .compatible = "ti,am654-mcspi",
1414                 .data = &am654_pdata,
1415         },
1416         { },
1417 };
1418 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1419
1420 static int omap2_mcspi_probe(struct platform_device *pdev)
1421 {
1422         struct spi_master       *master;
1423         const struct omap2_mcspi_platform_config *pdata;
1424         struct omap2_mcspi      *mcspi;
1425         struct resource         *r;
1426         int                     status = 0, i;
1427         u32                     regs_offset = 0;
1428         struct device_node      *node = pdev->dev.of_node;
1429         const struct of_device_id *match;
1430
1431         if (of_property_read_bool(node, "spi-slave"))
1432                 master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
1433         else
1434                 master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
1435         if (!master)
1436                 return -ENOMEM;
1437
1438         /* the spi->mode bits understood by this driver: */
1439         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1440         master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1441         master->setup = omap2_mcspi_setup;
1442         master->auto_runtime_pm = true;
1443         master->prepare_message = omap2_mcspi_prepare_message;
1444         master->can_dma = omap2_mcspi_can_dma;
1445         master->transfer_one = omap2_mcspi_transfer_one;
1446         master->set_cs = omap2_mcspi_set_cs;
1447         master->cleanup = omap2_mcspi_cleanup;
1448         master->slave_abort = omap2_mcspi_slave_abort;
1449         master->dev.of_node = node;
1450         master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1451         master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1452         master->use_gpio_descriptors = true;
1453
1454         platform_set_drvdata(pdev, master);
1455
1456         mcspi = spi_master_get_devdata(master);
1457         mcspi->master = master;
1458
1459         match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1460         if (match) {
1461                 u32 num_cs = 1; /* default number of chipselect */
1462                 pdata = match->data;
1463
1464                 of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1465                 master->num_chipselect = num_cs;
1466                 if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1467                         mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1468         } else {
1469                 pdata = dev_get_platdata(&pdev->dev);
1470                 master->num_chipselect = pdata->num_cs;
1471                 mcspi->pin_dir = pdata->pin_dir;
1472         }
1473         regs_offset = pdata->regs_offset;
1474         if (pdata->max_xfer_len) {
1475                 mcspi->max_xfer_len = pdata->max_xfer_len;
1476                 master->max_transfer_size = omap2_mcspi_max_xfer_size;
1477         }
1478
1479         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1480         mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1481         if (IS_ERR(mcspi->base)) {
1482                 status = PTR_ERR(mcspi->base);
1483                 goto free_master;
1484         }
1485         mcspi->phys = r->start + regs_offset;
1486         mcspi->base += regs_offset;
1487
1488         mcspi->dev = &pdev->dev;
1489
1490         INIT_LIST_HEAD(&mcspi->ctx.cs);
1491
1492         mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1493                                            sizeof(struct omap2_mcspi_dma),
1494                                            GFP_KERNEL);
1495         if (mcspi->dma_channels == NULL) {
1496                 status = -ENOMEM;
1497                 goto free_master;
1498         }
1499
1500         for (i = 0; i < master->num_chipselect; i++) {
1501                 sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1502                 sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1503
1504                 status = omap2_mcspi_request_dma(mcspi,
1505                                                  &mcspi->dma_channels[i]);
1506                 if (status == -EPROBE_DEFER)
1507                         goto free_master;
1508         }
1509
1510         status = platform_get_irq(pdev, 0);
1511         if (status == -EPROBE_DEFER)
1512                 goto free_master;
1513         if (status < 0) {
1514                 dev_err(&pdev->dev, "no irq resource found\n");
1515                 goto free_master;
1516         }
1517         init_completion(&mcspi->txdone);
1518         status = devm_request_irq(&pdev->dev, status,
1519                                   omap2_mcspi_irq_handler, 0, pdev->name,
1520                                   mcspi);
1521         if (status) {
1522                 dev_err(&pdev->dev, "Cannot request IRQ");
1523                 goto free_master;
1524         }
1525
1526         pm_runtime_use_autosuspend(&pdev->dev);
1527         pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1528         pm_runtime_enable(&pdev->dev);
1529
1530         status = omap2_mcspi_controller_setup(mcspi);
1531         if (status < 0)
1532                 goto disable_pm;
1533
1534         status = devm_spi_register_controller(&pdev->dev, master);
1535         if (status < 0)
1536                 goto disable_pm;
1537
1538         return status;
1539
1540 disable_pm:
1541         pm_runtime_dont_use_autosuspend(&pdev->dev);
1542         pm_runtime_put_sync(&pdev->dev);
1543         pm_runtime_disable(&pdev->dev);
1544 free_master:
1545         omap2_mcspi_release_dma(master);
1546         spi_master_put(master);
1547         return status;
1548 }
1549
1550 static int omap2_mcspi_remove(struct platform_device *pdev)
1551 {
1552         struct spi_master *master = platform_get_drvdata(pdev);
1553         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1554
1555         omap2_mcspi_release_dma(master);
1556
1557         pm_runtime_dont_use_autosuspend(mcspi->dev);
1558         pm_runtime_put_sync(mcspi->dev);
1559         pm_runtime_disable(&pdev->dev);
1560
1561         return 0;
1562 }
1563
1564 /* work with hotplug and coldplug */
1565 MODULE_ALIAS("platform:omap2_mcspi");
1566
1567 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1568 {
1569         struct spi_master *master = dev_get_drvdata(dev);
1570         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1571         int error;
1572
1573         error = pinctrl_pm_select_sleep_state(dev);
1574         if (error)
1575                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1576                          __func__, error);
1577
1578         error = spi_master_suspend(master);
1579         if (error)
1580                 dev_warn(mcspi->dev, "%s: master suspend failed: %i\n",
1581                          __func__, error);
1582
1583         return pm_runtime_force_suspend(dev);
1584 }
1585
1586 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1587 {
1588         struct spi_master *master = dev_get_drvdata(dev);
1589         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1590         int error;
1591
1592         error = spi_master_resume(master);
1593         if (error)
1594                 dev_warn(mcspi->dev, "%s: master resume failed: %i\n",
1595                          __func__, error);
1596
1597         return pm_runtime_force_resume(dev);
1598 }
1599
1600 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1601         SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1602                                 omap2_mcspi_resume)
1603         .runtime_suspend        = omap_mcspi_runtime_suspend,
1604         .runtime_resume         = omap_mcspi_runtime_resume,
1605 };
1606
1607 static struct platform_driver omap2_mcspi_driver = {
1608         .driver = {
1609                 .name =         "omap2_mcspi",
1610                 .pm =           &omap2_mcspi_pm_ops,
1611                 .of_match_table = omap_mcspi_of_match,
1612         },
1613         .probe =        omap2_mcspi_probe,
1614         .remove =       omap2_mcspi_remove,
1615 };
1616
1617 module_platform_driver(omap2_mcspi_driver);
1618 MODULE_LICENSE("GPL");