drivers/spi/spi-tegra210-quad.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 //
   3 // Copyright (C) 2020 NVIDIA CORPORATION.
   4
   5 #include <linux/clk.h>
   6 #include <linux/completion.h>
   7 #include <linux/delay.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/dmapool.h>
  11 #include <linux/err.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/iopoll.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kthread.h>
  17 #include <linux/module.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/pm_runtime.h>
  20 #include <linux/of.h>
  21 #include <linux/of_device.h>
  22 #include <linux/reset.h>
  23 #include <linux/spi/spi.h>
  24 #include <linux/acpi.h>
  25 #include <linux/property.h>
  26
  27 #define QSPI_COMMAND1                           0x000
  28 #define QSPI_BIT_LENGTH(x)                      (((x) & 0x1f) << 0)
  29 #define QSPI_PACKED                             BIT(5)
  30 #define QSPI_INTERFACE_WIDTH_MASK               (0x03 << 7)
  31 #define QSPI_INTERFACE_WIDTH(x)                 (((x) & 0x03) << 7)
  32 #define QSPI_INTERFACE_WIDTH_SINGLE             QSPI_INTERFACE_WIDTH(0)
  33 #define QSPI_INTERFACE_WIDTH_DUAL               QSPI_INTERFACE_WIDTH(1)
  34 #define QSPI_INTERFACE_WIDTH_QUAD               QSPI_INTERFACE_WIDTH(2)
  35 #define QSPI_SDR_DDR_SEL                        BIT(9)
  36 #define QSPI_TX_EN                              BIT(11)
  37 #define QSPI_RX_EN                              BIT(12)
  38 #define QSPI_CS_SW_VAL                          BIT(20)
  39 #define QSPI_CS_SW_HW                           BIT(21)
  40
  41 #define QSPI_CS_POL_INACTIVE(n)                 (1 << (22 + (n)))
  42 #define QSPI_CS_POL_INACTIVE_MASK               (0xF << 22)
  43 #define QSPI_CS_SEL_0                           (0 << 26)
  44 #define QSPI_CS_SEL_1                           (1 << 26)
  45 #define QSPI_CS_SEL_2                           (2 << 26)
  46 #define QSPI_CS_SEL_3                           (3 << 26)
  47 #define QSPI_CS_SEL_MASK                        (3 << 26)
  48 #define QSPI_CS_SEL(x)                          (((x) & 0x3) << 26)
  49
  50 #define QSPI_CONTROL_MODE_0                     (0 << 28)
  51 #define QSPI_CONTROL_MODE_3                     (3 << 28)
  52 #define QSPI_CONTROL_MODE_MASK                  (3 << 28)
  53 #define QSPI_M_S                                BIT(30)
  54 #define QSPI_PIO                                BIT(31)
  55
  56 #define QSPI_COMMAND2                           0x004
  57 #define QSPI_TX_TAP_DELAY(x)                    (((x) & 0x3f) << 10)
  58 #define QSPI_RX_TAP_DELAY(x)                    (((x) & 0xff) << 0)
  59
  60 #define QSPI_CS_TIMING1                         0x008
  61 #define QSPI_SETUP_HOLD(setup, hold)            (((setup) << 4) | (hold))
  62
  63 #define QSPI_CS_TIMING2                         0x00c
  64 #define CYCLES_BETWEEN_PACKETS_0(x)             (((x) & 0x1f) << 0)
  65 #define CS_ACTIVE_BETWEEN_PACKETS_0             BIT(5)
  66
  67 #define QSPI_TRANS_STATUS                       0x010
  68 #define QSPI_BLK_CNT(val)                       (((val) >> 0) & 0xffff)
  69 #define QSPI_RDY                                BIT(30)
  70
  71 #define QSPI_FIFO_STATUS                        0x014
  72 #define QSPI_RX_FIFO_EMPTY                      BIT(0)
  73 #define QSPI_RX_FIFO_FULL                       BIT(1)
  74 #define QSPI_TX_FIFO_EMPTY                      BIT(2)
  75 #define QSPI_TX_FIFO_FULL                       BIT(3)
  76 #define QSPI_RX_FIFO_UNF                        BIT(4)
  77 #define QSPI_RX_FIFO_OVF                        BIT(5)
  78 #define QSPI_TX_FIFO_UNF                        BIT(6)
  79 #define QSPI_TX_FIFO_OVF                        BIT(7)
  80 #define QSPI_ERR                                BIT(8)
  81 #define QSPI_TX_FIFO_FLUSH                      BIT(14)
  82 #define QSPI_RX_FIFO_FLUSH                      BIT(15)
  83 #define QSPI_TX_FIFO_EMPTY_COUNT(val)           (((val) >> 16) & 0x7f)
  84 #define QSPI_RX_FIFO_FULL_COUNT(val)            (((val) >> 23) & 0x7f)
  85
  86 #define QSPI_FIFO_ERROR                         (QSPI_RX_FIFO_UNF | \
  87                                                  QSPI_RX_FIFO_OVF | \
  88                                                  QSPI_TX_FIFO_UNF | \
  89                                                  QSPI_TX_FIFO_OVF)
  90 #define QSPI_FIFO_EMPTY                         (QSPI_RX_FIFO_EMPTY | \
  91                                                  QSPI_TX_FIFO_EMPTY)
  92
  93 #define QSPI_TX_DATA                            0x018
  94 #define QSPI_RX_DATA                            0x01c
  95
  96 #define QSPI_DMA_CTL                            0x020
  97 #define QSPI_TX_TRIG(n)                         (((n) & 0x3) << 15)
  98 #define QSPI_TX_TRIG_1                          QSPI_TX_TRIG(0)
  99 #define QSPI_TX_TRIG_4                          QSPI_TX_TRIG(1)
 100 #define QSPI_TX_TRIG_8                          QSPI_TX_TRIG(2)
 101 #define QSPI_TX_TRIG_16                         QSPI_TX_TRIG(3)
 102
 103 #define QSPI_RX_TRIG(n)                         (((n) & 0x3) << 19)
 104 #define QSPI_RX_TRIG_1                          QSPI_RX_TRIG(0)
 105 #define QSPI_RX_TRIG_4                          QSPI_RX_TRIG(1)
 106 #define QSPI_RX_TRIG_8                          QSPI_RX_TRIG(2)
 107 #define QSPI_RX_TRIG_16                         QSPI_RX_TRIG(3)
 108
 109 #define QSPI_DMA_EN                             BIT(31)
 110
 111 #define QSPI_DMA_BLK                            0x024
 112 #define QSPI_DMA_BLK_SET(x)                     (((x) & 0xffff) << 0)
 113
 114 #define QSPI_TX_FIFO                            0x108
 115 #define QSPI_RX_FIFO                            0x188
 116
 117 #define QSPI_FIFO_DEPTH                         64
 118
 119 #define QSPI_INTR_MASK                          0x18c
 120 #define QSPI_INTR_RX_FIFO_UNF_MASK              BIT(25)
 121 #define QSPI_INTR_RX_FIFO_OVF_MASK              BIT(26)
 122 #define QSPI_INTR_TX_FIFO_UNF_MASK              BIT(27)
 123 #define QSPI_INTR_TX_FIFO_OVF_MASK              BIT(28)
 124 #define QSPI_INTR_RDY_MASK                      BIT(29)
 125 #define QSPI_INTR_RX_TX_FIFO_ERR                (QSPI_INTR_RX_FIFO_UNF_MASK | \
 126                                                  QSPI_INTR_RX_FIFO_OVF_MASK | \
 127                                                  QSPI_INTR_TX_FIFO_UNF_MASK | \
 128                                                  QSPI_INTR_TX_FIFO_OVF_MASK)
 129
 130 #define QSPI_MISC_REG                           0x194
 131 #define QSPI_NUM_DUMMY_CYCLE(x)                 (((x) & 0xff) << 0)
 132 #define QSPI_DUMMY_CYCLES_MAX                   0xff
 133
 134 #define QSPI_CMB_SEQ_CMD                        0x19c
 135 #define QSPI_COMMAND_VALUE_SET(X)               (((x) & 0xFF) << 0)
 136
 137 #define QSPI_CMB_SEQ_CMD_CFG                    0x1a0
 138 #define QSPI_COMMAND_X1_X2_X4(x)                (((x) & 0x3) << 13)
 139 #define QSPI_COMMAND_X1_X2_X4_MASK              (0x03 << 13)
 140 #define QSPI_COMMAND_SDR_DDR                    BIT(12)
 141 #define QSPI_COMMAND_SIZE_SET(x)                (((x) & 0xFF) << 0)
 142
 143 #define QSPI_GLOBAL_CONFIG                      0X1a4
 144 #define QSPI_CMB_SEQ_EN                         BIT(0)
 145 #define QSPI_TPM_WAIT_POLL_EN                   BIT(1)
 146
 147 #define QSPI_CMB_SEQ_ADDR                       0x1a8
 148 #define QSPI_ADDRESS_VALUE_SET(X)               (((x) & 0xFFFF) << 0)
 149
 150 #define QSPI_CMB_SEQ_ADDR_CFG                   0x1ac
 151 #define QSPI_ADDRESS_X1_X2_X4(x)                (((x) & 0x3) << 13)
 152 #define QSPI_ADDRESS_X1_X2_X4_MASK              (0x03 << 13)
 153 #define QSPI_ADDRESS_SDR_DDR                    BIT(12)
 154 #define QSPI_ADDRESS_SIZE_SET(x)                (((x) & 0xFF) << 0)
 155
 156 #define DATA_DIR_TX                             BIT(0)
 157 #define DATA_DIR_RX                             BIT(1)
 158
 159 #define QSPI_DMA_TIMEOUT                        (msecs_to_jiffies(1000))
 160 #define DEFAULT_QSPI_DMA_BUF_LEN                (64 * 1024)
 161 #define CMD_TRANSFER                            0
 162 #define ADDR_TRANSFER                           1
 163 #define DATA_TRANSFER                           2
 164
 165 struct tegra_qspi_soc_data {
 166         bool has_dma;
 167         bool cmb_xfer_capable;
 168         bool supports_tpm;
 169         unsigned int cs_count;
 170 };
 171
 172 struct tegra_qspi_client_data {
 173         int tx_clk_tap_delay;
 174         int rx_clk_tap_delay;
 175 };
 176
 177 struct tegra_qspi {
 178         struct device                           *dev;
 179         struct spi_master                       *master;
 180         /* lock to protect data accessed by irq */
 181         spinlock_t                              lock;
 182
 183         struct clk                              *clk;
 184         void __iomem                            *base;
 185         phys_addr_t                             phys;
 186         unsigned int                            irq;
 187
 188         u32                                     cur_speed;
 189         unsigned int                            cur_pos;
 190         unsigned int                            words_per_32bit;
 191         unsigned int                            bytes_per_word;
 192         unsigned int                            curr_dma_words;
 193         unsigned int                            cur_direction;
 194
 195         unsigned int                            cur_rx_pos;
 196         unsigned int                            cur_tx_pos;
 197
 198         unsigned int                            dma_buf_size;
 199         unsigned int                            max_buf_size;
 200         bool                                    is_curr_dma_xfer;
 201
 202         struct completion                       rx_dma_complete;
 203         struct completion                       tx_dma_complete;
 204
 205         u32                                     tx_status;
 206         u32                                     rx_status;
 207         u32                                     status_reg;
 208         bool                                    is_packed;
 209         bool                                    use_dma;
 210
 211         u32                                     command1_reg;
 212         u32                                     dma_control_reg;
 213         u32                                     def_command1_reg;
 214         u32                                     def_command2_reg;
 215         u32                                     spi_cs_timing1;
 216         u32                                     spi_cs_timing2;
 217         u8                                      dummy_cycles;
 218
 219         struct completion                       xfer_completion;
 220         struct spi_transfer                     *curr_xfer;
 221
 222         struct dma_chan                         *rx_dma_chan;
 223         u32                                     *rx_dma_buf;
 224         dma_addr_t                              rx_dma_phys;
 225         struct dma_async_tx_descriptor          *rx_dma_desc;
 226
 227         struct dma_chan                         *tx_dma_chan;
 228         u32                                     *tx_dma_buf;
 229         dma_addr_t                              tx_dma_phys;
 230         struct dma_async_tx_descriptor          *tx_dma_desc;
 231         const struct tegra_qspi_soc_data        *soc_data;
 232 };
 233
 234 static inline u32 tegra_qspi_readl(struct tegra_qspi *tqspi, unsigned long offset)
 235 {
 236         return readl(tqspi->base + offset);
 237 }
 238
 239 static inline void tegra_qspi_writel(struct tegra_qspi *tqspi, u32 value, unsigned long offset)
 240 {
 241         writel(value, tqspi->base + offset);
 242
 243         /* read back register to make sure that register writes completed */
 244         if (offset != QSPI_TX_FIFO)
 245                 readl(tqspi->base + QSPI_COMMAND1);
 246 }
 247
 248 static void tegra_qspi_mask_clear_irq(struct tegra_qspi *tqspi)
 249 {
 250         u32 value;
 251
 252         /* write 1 to clear status register */
 253         value = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
 254         tegra_qspi_writel(tqspi, value, QSPI_TRANS_STATUS);
 255
 256         value = tegra_qspi_readl(tqspi, QSPI_INTR_MASK);
 257         if (!(value & QSPI_INTR_RDY_MASK)) {
 258                 value |= (QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR);
 259                 tegra_qspi_writel(tqspi, value, QSPI_INTR_MASK);
 260         }
 261
 262         /* clear fifo status error if any */
 263         value = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
 264         if (value & QSPI_ERR)
 265                 tegra_qspi_writel(tqspi, QSPI_ERR | QSPI_FIFO_ERROR, QSPI_FIFO_STATUS);
 266 }
 267
 268 static unsigned int
 269 tegra_qspi_calculate_curr_xfer_param(struct tegra_qspi *tqspi, struct spi_transfer *t)
 270 {
 271         unsigned int max_word, max_len, total_fifo_words;
 272         unsigned int remain_len = t->len - tqspi->cur_pos;
 273         unsigned int bits_per_word = t->bits_per_word;
 274
 275         tqspi->bytes_per_word = DIV_ROUND_UP(bits_per_word, 8);
 276
 277         /*
 278          * Tegra QSPI controller supports packed or unpacked mode transfers.
 279          * Packed mode is used for data transfers using 8, 16, or 32 bits per
 280          * word with a minimum transfer of 1 word and for all other transfers
 281          * unpacked mode will be used.
 282          */
 283
 284         if ((bits_per_word == 8 || bits_per_word == 16 ||
 285              bits_per_word == 32) && t->len > 3) {
 286                 tqspi->is_packed = true;
 287                 tqspi->words_per_32bit = 32 / bits_per_word;
 288         } else {
 289                 tqspi->is_packed = false;
 290                 tqspi->words_per_32bit = 1;
 291         }
 292
 293         if (tqspi->is_packed) {
 294                 max_len = min(remain_len, tqspi->max_buf_size);
 295                 tqspi->curr_dma_words = max_len / tqspi->bytes_per_word;
 296                 total_fifo_words = (max_len + 3) / 4;
 297         } else {
 298                 max_word = (remain_len - 1) / tqspi->bytes_per_word + 1;
 299                 max_word = min(max_word, tqspi->max_buf_size / 4);
 300                 tqspi->curr_dma_words = max_word;
 301                 total_fifo_words = max_word;
 302         }
 303
 304         return total_fifo_words;
 305 }
 306
 307 static unsigned int
 308 tegra_qspi_fill_tx_fifo_from_client_txbuf(struct tegra_qspi *tqspi, struct spi_transfer *t)
 309 {
 310         unsigned int written_words, fifo_words_left, count;
 311         unsigned int len, tx_empty_count, max_n_32bit, i;
 312         u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos;
 313         u32 fifo_status;
 314
 315         fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
 316         tx_empty_count = QSPI_TX_FIFO_EMPTY_COUNT(fifo_status);
 317
 318         if (tqspi->is_packed) {
 319                 fifo_words_left = tx_empty_count * tqspi->words_per_32bit;
 320                 written_words = min(fifo_words_left, tqspi->curr_dma_words);
 321                 len = written_words * tqspi->bytes_per_word;
 322                 max_n_32bit = DIV_ROUND_UP(len, 4);
 323                 for (count = 0; count < max_n_32bit; count++) {
 324                         u32 x = 0;
 325
 326                         for (i = 0; (i < 4) && len; i++, len--)
 327                                 x |= (u32)(*tx_buf++) << (i * 8);
 328                         tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO);
 329                 }
 330
 331                 tqspi->cur_tx_pos += written_words * tqspi->bytes_per_word;
 332         } else {
 333                 unsigned int write_bytes;
 334                 u8 bytes_per_word = tqspi->bytes_per_word;
 335
 336                 max_n_32bit = min(tqspi->curr_dma_words, tx_empty_count);
 337                 written_words = max_n_32bit;
 338                 len = written_words * tqspi->bytes_per_word;
 339                 if (len > t->len - tqspi->cur_pos)
 340                         len = t->len - tqspi->cur_pos;
 341                 write_bytes = len;
 342                 for (count = 0; count < max_n_32bit; count++) {
 343                         u32 x = 0;
 344
 345                         for (i = 0; len && (i < bytes_per_word); i++, len--)
 346                                 x |= (u32)(*tx_buf++) << (i * 8);
 347                         tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO);
 348                 }
 349
 350                 tqspi->cur_tx_pos += write_bytes;
 351         }
 352
 353         return written_words;
 354 }
 355
 356 static unsigned int
 357 tegra_qspi_read_rx_fifo_to_client_rxbuf(struct tegra_qspi *tqspi, struct spi_transfer *t)
 358 {
 359         u8 *rx_buf = (u8 *)t->rx_buf + tqspi->cur_rx_pos;
 360         unsigned int len, rx_full_count, count, i;
 361         unsigned int read_words = 0;
 362         u32 fifo_status, x;
 363
 364         fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
 365         rx_full_count = QSPI_RX_FIFO_FULL_COUNT(fifo_status);
 366         if (tqspi->is_packed) {
 367                 len = tqspi->curr_dma_words * tqspi->bytes_per_word;
 368                 for (count = 0; count < rx_full_count; count++) {
 369                         x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO);
 370
 371                         for (i = 0; len && (i < 4); i++, len--)
 372                                 *rx_buf++ = (x >> i * 8) & 0xff;
 373                 }
 374
 375                 read_words += tqspi->curr_dma_words;
 376                 tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word;
 377         } else {
 378                 u32 rx_mask = ((u32)1 << t->bits_per_word) - 1;
 379                 u8 bytes_per_word = tqspi->bytes_per_word;
 380                 unsigned int read_bytes;
 381
 382                 len = rx_full_count * bytes_per_word;
 383                 if (len > t->len - tqspi->cur_pos)
 384                         len = t->len - tqspi->cur_pos;
 385                 read_bytes = len;
 386                 for (count = 0; count < rx_full_count; count++) {
 387                         x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO) & rx_mask;
 388
 389                         for (i = 0; len && (i < bytes_per_word); i++, len--)
 390                                 *rx_buf++ = (x >> (i * 8)) & 0xff;
 391                 }
 392
 393                 read_words += rx_full_count;
 394                 tqspi->cur_rx_pos += read_bytes;
 395         }
 396
 397         return read_words;
 398 }
 399
 400 static void
 401 tegra_qspi_copy_client_txbuf_to_qspi_txbuf(struct tegra_qspi *tqspi, struct spi_transfer *t)
 402 {
 403         dma_sync_single_for_cpu(tqspi->dev, tqspi->tx_dma_phys,
 404                                 tqspi->dma_buf_size, DMA_TO_DEVICE);
 405
 406         /*
 407          * In packed mode, each word in FIFO may contain multiple packets
 408          * based on bits per word. So all bytes in each FIFO word are valid.
 409          *
 410          * In unpacked mode, each word in FIFO contains single packet and
 411          * based on bits per word any remaining bits in FIFO word will be
 412          * ignored by the hardware and are invalid bits.
 413          */
 414         if (tqspi->is_packed) {
 415                 tqspi->cur_tx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word;
 416         } else {
 417                 u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos;
 418                 unsigned int i, count, consume, write_bytes;
 419
 420                 /*
 421                  * Fill tx_dma_buf to contain single packet in each word based
 422                  * on bits per word from SPI core tx_buf.
 423                  */
 424                 consume = tqspi->curr_dma_words * tqspi->bytes_per_word;
 425                 if (consume > t->len - tqspi->cur_pos)
 426                         consume = t->len - tqspi->cur_pos;
 427                 write_bytes = consume;
 428                 for (count = 0; count < tqspi->curr_dma_words; count++) {
 429                         u32 x = 0;
 430
 431                         for (i = 0; consume && (i < tqspi->bytes_per_word); i++, consume--)
 432                                 x |= (u32)(*tx_buf++) << (i * 8);
 433                         tqspi->tx_dma_buf[count] = x;
 434                 }
 435
 436                 tqspi->cur_tx_pos += write_bytes;
 437         }
 438
 439         dma_sync_single_for_device(tqspi->dev, tqspi->tx_dma_phys,
 440                                    tqspi->dma_buf_size, DMA_TO_DEVICE);
 441 }
 442
 443 static void
 444 tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf(struct tegra_qspi *tqspi, struct spi_transfer *t)
 445 {
 446         dma_sync_single_for_cpu(tqspi->dev, tqspi->rx_dma_phys,
 447                                 tqspi->dma_buf_size, DMA_FROM_DEVICE);
 448
 449         if (tqspi->is_packed) {
 450                 tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word;
 451         } else {
 452                 unsigned char *rx_buf = t->rx_buf + tqspi->cur_rx_pos;
 453                 u32 rx_mask = ((u32)1 << t->bits_per_word) - 1;
 454                 unsigned int i, count, consume, read_bytes;
 455
 456                 /*
 457                  * Each FIFO word contains single data packet.
 458                  * Skip invalid bits in each FIFO word based on bits per word
 459                  * and align bytes while filling in SPI core rx_buf.
 460                  */
 461                 consume = tqspi->curr_dma_words * tqspi->bytes_per_word;
 462                 if (consume > t->len - tqspi->cur_pos)
 463                         consume = t->len - tqspi->cur_pos;
 464                 read_bytes = consume;
 465                 for (count = 0; count < tqspi->curr_dma_words; count++) {
 466                         u32 x = tqspi->rx_dma_buf[count] & rx_mask;
 467
 468                         for (i = 0; consume && (i < tqspi->bytes_per_word); i++, consume--)
 469                                 *rx_buf++ = (x >> (i * 8)) & 0xff;
 470                 }
 471
 472                 tqspi->cur_rx_pos += read_bytes;
 473         }
 474
 475         dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys,
 476                                    tqspi->dma_buf_size, DMA_FROM_DEVICE);
 477 }
 478
 479 static void tegra_qspi_dma_complete(void *args)
 480 {
 481         struct completion *dma_complete = args;
 482
 483         complete(dma_complete);
 484 }
 485
 486 static int tegra_qspi_start_tx_dma(struct tegra_qspi *tqspi, struct spi_transfer *t, int len)
 487 {
 488         dma_addr_t tx_dma_phys;
 489
 490         reinit_completion(&tqspi->tx_dma_complete);
 491
 492         if (tqspi->is_packed)
 493                 tx_dma_phys = t->tx_dma;
 494         else
 495                 tx_dma_phys = tqspi->tx_dma_phys;
 496
 497         tqspi->tx_dma_desc = dmaengine_prep_slave_single(tqspi->tx_dma_chan, tx_dma_phys,
 498                                                          len, DMA_MEM_TO_DEV,
 499                                                          DMA_PREP_INTERRUPT |  DMA_CTRL_ACK);
 500
 501         if (!tqspi->tx_dma_desc) {
 502                 dev_err(tqspi->dev, "Unable to get TX descriptor\n");
 503                 return -EIO;
 504         }
 505
 506         tqspi->tx_dma_desc->callback = tegra_qspi_dma_complete;
 507         tqspi->tx_dma_desc->callback_param = &tqspi->tx_dma_complete;
 508         dmaengine_submit(tqspi->tx_dma_desc);
 509         dma_async_issue_pending(tqspi->tx_dma_chan);
 510
 511         return 0;
 512 }
 513
 514 static int tegra_qspi_start_rx_dma(struct tegra_qspi *tqspi, struct spi_transfer *t, int len)
 515 {
 516         dma_addr_t rx_dma_phys;
 517
 518         reinit_completion(&tqspi->rx_dma_complete);
 519
 520         if (tqspi->is_packed)
 521                 rx_dma_phys = t->rx_dma;
 522         else
 523                 rx_dma_phys = tqspi->rx_dma_phys;
 524
 525         tqspi->rx_dma_desc = dmaengine_prep_slave_single(tqspi->rx_dma_chan, rx_dma_phys,
 526                                                          len, DMA_DEV_TO_MEM,
 527                                                          DMA_PREP_INTERRUPT |  DMA_CTRL_ACK);
 528
 529         if (!tqspi->rx_dma_desc) {
 530                 dev_err(tqspi->dev, "Unable to get RX descriptor\n");
 531                 return -EIO;
 532         }
 533
 534         tqspi->rx_dma_desc->callback = tegra_qspi_dma_complete;
 535         tqspi->rx_dma_desc->callback_param = &tqspi->rx_dma_complete;
 536         dmaengine_submit(tqspi->rx_dma_desc);
 537         dma_async_issue_pending(tqspi->rx_dma_chan);
 538
 539         return 0;
 540 }
 541
 542 static int tegra_qspi_flush_fifos(struct tegra_qspi *tqspi, bool atomic)
 543 {
 544         void __iomem *addr = tqspi->base + QSPI_FIFO_STATUS;
 545         u32 val;
 546
 547         val = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
 548         if ((val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY)
 549                 return 0;
 550
 551         val |= QSPI_RX_FIFO_FLUSH | QSPI_TX_FIFO_FLUSH;
 552         tegra_qspi_writel(tqspi, val, QSPI_FIFO_STATUS);
 553
 554         if (!atomic)
 555                 return readl_relaxed_poll_timeout(addr, val,
 556                                                   (val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY,
 557                                                   1000, 1000000);
 558
 559         return readl_relaxed_poll_timeout_atomic(addr, val,
 560                                                  (val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY,
 561                                                  1000, 1000000);
 562 }
 563
 564 static void tegra_qspi_unmask_irq(struct tegra_qspi *tqspi)
 565 {
 566         u32 intr_mask;
 567
 568         intr_mask = tegra_qspi_readl(tqspi, QSPI_INTR_MASK);
 569         intr_mask &= ~(QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR);
 570         tegra_qspi_writel(tqspi, intr_mask, QSPI_INTR_MASK);
 571 }
 572
 573 static int tegra_qspi_dma_map_xfer(struct tegra_qspi *tqspi, struct spi_transfer *t)
 574 {
 575         u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos;
 576         u8 *rx_buf = (u8 *)t->rx_buf + tqspi->cur_rx_pos;
 577         unsigned int len;
 578
 579         len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4;
 580
 581         if (t->tx_buf) {
 582                 t->tx_dma = dma_map_single(tqspi->dev, (void *)tx_buf, len, DMA_TO_DEVICE);
 583                 if (dma_mapping_error(tqspi->dev, t->tx_dma))
 584                         return -ENOMEM;
 585         }
 586
 587         if (t->rx_buf) {
 588                 t->rx_dma = dma_map_single(tqspi->dev, (void *)rx_buf, len, DMA_FROM_DEVICE);
 589                 if (dma_mapping_error(tqspi->dev, t->rx_dma)) {
 590                         dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE);
 591                         return -ENOMEM;
 592                 }
 593         }
 594
 595         return 0;
 596 }
 597
 598 static void tegra_qspi_dma_unmap_xfer(struct tegra_qspi *tqspi, struct spi_transfer *t)
 599 {
 600         unsigned int len;
 601
 602         len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4;
 603
 604         dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE);
 605         dma_unmap_single(tqspi->dev, t->rx_dma, len, DMA_FROM_DEVICE);
 606 }
 607
 608 static int tegra_qspi_start_dma_based_transfer(struct tegra_qspi *tqspi, struct spi_transfer *t)
 609 {
 610         struct dma_slave_config dma_sconfig = { 0 };
 611         unsigned int len;
 612         u8 dma_burst;
 613         int ret = 0;
 614         u32 val;
 615
 616         if (tqspi->is_packed) {
 617                 ret = tegra_qspi_dma_map_xfer(tqspi, t);
 618                 if (ret < 0)
 619                         return ret;
 620         }
 621
 622         val = QSPI_DMA_BLK_SET(tqspi->curr_dma_words - 1);
 623         tegra_qspi_writel(tqspi, val, QSPI_DMA_BLK);
 624
 625         tegra_qspi_unmask_irq(tqspi);
 626
 627         if (tqspi->is_packed)
 628                 len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4;
 629         else
 630                 len = tqspi->curr_dma_words * 4;
 631
 632         /* set attention level based on length of transfer */
 633         val = 0;
 634         if (len & 0xf) {
 635                 val |= QSPI_TX_TRIG_1 | QSPI_RX_TRIG_1;
 636                 dma_burst = 1;
 637         } else if (((len) >> 4) & 0x1) {
 638                 val |= QSPI_TX_TRIG_4 | QSPI_RX_TRIG_4;
 639                 dma_burst = 4;
 640         } else {
 641                 val |= QSPI_TX_TRIG_8 | QSPI_RX_TRIG_8;
 642                 dma_burst = 8;
 643         }
 644
 645         tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL);
 646         tqspi->dma_control_reg = val;
 647
 648         dma_sconfig.device_fc = true;
 649         if (tqspi->cur_direction & DATA_DIR_TX) {
 650                 dma_sconfig.dst_addr = tqspi->phys + QSPI_TX_FIFO;
 651                 dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 652                 dma_sconfig.dst_maxburst = dma_burst;
 653                 ret = dmaengine_slave_config(tqspi->tx_dma_chan, &dma_sconfig);
 654                 if (ret < 0) {
 655                         dev_err(tqspi->dev, "failed DMA slave config: %d\n", ret);
 656                         return ret;
 657                 }
 658
 659                 tegra_qspi_copy_client_txbuf_to_qspi_txbuf(tqspi, t);
 660                 ret = tegra_qspi_start_tx_dma(tqspi, t, len);
 661                 if (ret < 0) {
 662                         dev_err(tqspi->dev, "failed to starting TX DMA: %d\n", ret);
 663                         return ret;
 664                 }
 665         }
 666
 667         if (tqspi->cur_direction & DATA_DIR_RX) {
 668                 dma_sconfig.src_addr = tqspi->phys + QSPI_RX_FIFO;
 669                 dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 670                 dma_sconfig.src_maxburst = dma_burst;
 671                 ret = dmaengine_slave_config(tqspi->rx_dma_chan, &dma_sconfig);
 672                 if (ret < 0) {
 673                         dev_err(tqspi->dev, "failed DMA slave config: %d\n", ret);
 674                         return ret;
 675                 }
 676
 677                 dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys,
 678                                            tqspi->dma_buf_size,
 679                                            DMA_FROM_DEVICE);
 680
 681                 ret = tegra_qspi_start_rx_dma(tqspi, t, len);
 682                 if (ret < 0) {
 683                         dev_err(tqspi->dev, "failed to start RX DMA: %d\n", ret);
 684                         if (tqspi->cur_direction & DATA_DIR_TX)
 685                                 dmaengine_terminate_all(tqspi->tx_dma_chan);
 686                         return ret;
 687                 }
 688         }
 689
 690         tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1);
 691
 692         tqspi->is_curr_dma_xfer = true;
 693         tqspi->dma_control_reg = val;
 694         val |= QSPI_DMA_EN;
 695         tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL);
 696
 697         return ret;
 698 }
 699
 700 static int tegra_qspi_start_cpu_based_transfer(struct tegra_qspi *qspi, struct spi_transfer *t)
 701 {
 702         u32 val;
 703         unsigned int cur_words;
 704
 705         if (qspi->cur_direction & DATA_DIR_TX)
 706                 cur_words = tegra_qspi_fill_tx_fifo_from_client_txbuf(qspi, t);
 707         else
 708                 cur_words = qspi->curr_dma_words;
 709
 710         val = QSPI_DMA_BLK_SET(cur_words - 1);
 711         tegra_qspi_writel(qspi, val, QSPI_DMA_BLK);
 712
 713         tegra_qspi_unmask_irq(qspi);
 714
 715         qspi->is_curr_dma_xfer = false;
 716         val = qspi->command1_reg;
 717         val |= QSPI_PIO;
 718         tegra_qspi_writel(qspi, val, QSPI_COMMAND1);
 719
 720         return 0;
 721 }
 722
 723 static void tegra_qspi_deinit_dma(struct tegra_qspi *tqspi)
 724 {
 725         if (!tqspi->soc_data->has_dma)
 726                 return;
 727
 728         if (tqspi->tx_dma_buf) {
 729                 dma_free_coherent(tqspi->dev, tqspi->dma_buf_size,
 730                                   tqspi->tx_dma_buf, tqspi->tx_dma_phys);
 731                 tqspi->tx_dma_buf = NULL;
 732         }
 733
 734         if (tqspi->tx_dma_chan) {
 735                 dma_release_channel(tqspi->tx_dma_chan);
 736                 tqspi->tx_dma_chan = NULL;
 737         }
 738
 739         if (tqspi->rx_dma_buf) {
 740                 dma_free_coherent(tqspi->dev, tqspi->dma_buf_size,
 741                                   tqspi->rx_dma_buf, tqspi->rx_dma_phys);
 742                 tqspi->rx_dma_buf = NULL;
 743         }
 744
 745         if (tqspi->rx_dma_chan) {
 746                 dma_release_channel(tqspi->rx_dma_chan);
 747                 tqspi->rx_dma_chan = NULL;
 748         }
 749 }
 750
 751 static int tegra_qspi_init_dma(struct tegra_qspi *tqspi)
 752 {
 753         struct dma_chan *dma_chan;
 754         dma_addr_t dma_phys;
 755         u32 *dma_buf;
 756         int err;
 757
 758         if (!tqspi->soc_data->has_dma)
 759                 return 0;
 760
 761         dma_chan = dma_request_chan(tqspi->dev, "rx");
 762         if (IS_ERR(dma_chan)) {
 763                 err = PTR_ERR(dma_chan);
 764                 goto err_out;
 765         }
 766
 767         tqspi->rx_dma_chan = dma_chan;
 768
 769         dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL);
 770         if (!dma_buf) {
 771                 err = -ENOMEM;
 772                 goto err_out;
 773         }
 774
 775         tqspi->rx_dma_buf = dma_buf;
 776         tqspi->rx_dma_phys = dma_phys;
 777
 778         dma_chan = dma_request_chan(tqspi->dev, "tx");
 779         if (IS_ERR(dma_chan)) {
 780                 err = PTR_ERR(dma_chan);
 781                 goto err_out;
 782         }
 783
 784         tqspi->tx_dma_chan = dma_chan;
 785
 786         dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL);
 787         if (!dma_buf) {
 788                 err = -ENOMEM;
 789                 goto err_out;
 790         }
 791
 792         tqspi->tx_dma_buf = dma_buf;
 793         tqspi->tx_dma_phys = dma_phys;
 794         tqspi->use_dma = true;
 795
 796         return 0;
 797
 798 err_out:
 799         tegra_qspi_deinit_dma(tqspi);
 800
 801         if (err != -EPROBE_DEFER) {
 802                 dev_err(tqspi->dev, "cannot use DMA: %d\n", err);
 803                 dev_err(tqspi->dev, "falling back to PIO\n");
 804                 return 0;
 805         }
 806
 807         return err;
 808 }
 809
 810 static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_transfer *t,
 811                                          bool is_first_of_msg)
 812 {
 813         struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master);
 814         struct tegra_qspi_client_data *cdata = spi->controller_data;
 815         u32 command1, command2, speed = t->speed_hz;
 816         u8 bits_per_word = t->bits_per_word;
 817         u32 tx_tap = 0, rx_tap = 0;
 818         int req_mode;
 819
 820         if (!has_acpi_companion(tqspi->dev) && speed != tqspi->cur_speed) {
 821                 clk_set_rate(tqspi->clk, speed);
 822                 tqspi->cur_speed = speed;
 823         }
 824
 825         tqspi->cur_pos = 0;
 826         tqspi->cur_rx_pos = 0;
 827         tqspi->cur_tx_pos = 0;
 828         tqspi->curr_xfer = t;
 829
 830         if (is_first_of_msg) {
 831                 tegra_qspi_mask_clear_irq(tqspi);
 832
 833                 command1 = tqspi->def_command1_reg;
 834                 command1 |= QSPI_CS_SEL(spi_get_chipselect(spi, 0));
 835                 command1 |= QSPI_BIT_LENGTH(bits_per_word - 1);
 836
 837                 command1 &= ~QSPI_CONTROL_MODE_MASK;
 838                 req_mode = spi->mode & 0x3;
 839                 if (req_mode == SPI_MODE_3)
 840                         command1 |= QSPI_CONTROL_MODE_3;
 841                 else
 842                         command1 |= QSPI_CONTROL_MODE_0;
 843
 844                 if (spi->mode & SPI_CS_HIGH)
 845                         command1 |= QSPI_CS_SW_VAL;
 846                 else
 847                         command1 &= ~QSPI_CS_SW_VAL;
 848                 tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1);
 849
 850                 if (cdata && cdata->tx_clk_tap_delay)
 851                         tx_tap = cdata->tx_clk_tap_delay;
 852
 853                 if (cdata && cdata->rx_clk_tap_delay)
 854                         rx_tap = cdata->rx_clk_tap_delay;
 855
 856                 command2 = QSPI_TX_TAP_DELAY(tx_tap) | QSPI_RX_TAP_DELAY(rx_tap);
 857                 if (command2 != tqspi->def_command2_reg)
 858                         tegra_qspi_writel(tqspi, command2, QSPI_COMMAND2);
 859
 860         } else {
 861                 command1 = tqspi->command1_reg;
 862                 command1 &= ~QSPI_BIT_LENGTH(~0);
 863                 command1 |= QSPI_BIT_LENGTH(bits_per_word - 1);
 864         }
 865
 866         command1 &= ~QSPI_SDR_DDR_SEL;
 867
 868         return command1;
 869 }
 870
 871 static int tegra_qspi_start_transfer_one(struct spi_device *spi,
 872                                          struct spi_transfer *t, u32 command1)
 873 {
 874         struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master);
 875         unsigned int total_fifo_words;
 876         u8 bus_width = 0;
 877         int ret;
 878
 879         total_fifo_words = tegra_qspi_calculate_curr_xfer_param(tqspi, t);
 880
 881         command1 &= ~QSPI_PACKED;
 882         if (tqspi->is_packed)
 883                 command1 |= QSPI_PACKED;
 884         tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1);
 885
 886         tqspi->cur_direction = 0;
 887
 888         command1 &= ~(QSPI_TX_EN | QSPI_RX_EN);
 889         if (t->rx_buf) {
 890                 command1 |= QSPI_RX_EN;
 891                 tqspi->cur_direction |= DATA_DIR_RX;
 892                 bus_width = t->rx_nbits;
 893         }
 894
 895         if (t->tx_buf) {
 896                 command1 |= QSPI_TX_EN;
 897                 tqspi->cur_direction |= DATA_DIR_TX;
 898                 bus_width = t->tx_nbits;
 899         }
 900
 901         command1 &= ~QSPI_INTERFACE_WIDTH_MASK;
 902
 903         if (bus_width == SPI_NBITS_QUAD)
 904                 command1 |= QSPI_INTERFACE_WIDTH_QUAD;
 905         else if (bus_width == SPI_NBITS_DUAL)
 906                 command1 |= QSPI_INTERFACE_WIDTH_DUAL;
 907         else
 908                 command1 |= QSPI_INTERFACE_WIDTH_SINGLE;
 909
 910         tqspi->command1_reg = command1;
 911
 912         tegra_qspi_writel(tqspi, QSPI_NUM_DUMMY_CYCLE(tqspi->dummy_cycles), QSPI_MISC_REG);
 913
 914         ret = tegra_qspi_flush_fifos(tqspi, false);
 915         if (ret < 0)
 916                 return ret;
 917
 918         if (tqspi->use_dma && total_fifo_words > QSPI_FIFO_DEPTH)
 919                 ret = tegra_qspi_start_dma_based_transfer(tqspi, t);
 920         else
 921                 ret = tegra_qspi_start_cpu_based_transfer(tqspi, t);
 922
 923         return ret;
 924 }
 925
 926 static struct tegra_qspi_client_data *tegra_qspi_parse_cdata_dt(struct spi_device *spi)
 927 {
 928         struct tegra_qspi_client_data *cdata;
 929         struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master);
 930
 931         cdata = devm_kzalloc(tqspi->dev, sizeof(*cdata), GFP_KERNEL);
 932         if (!cdata)
 933                 return NULL;
 934
 935         device_property_read_u32(&spi->dev, "nvidia,tx-clk-tap-delay",
 936                                  &cdata->tx_clk_tap_delay);
 937         device_property_read_u32(&spi->dev, "nvidia,rx-clk-tap-delay",
 938                                  &cdata->rx_clk_tap_delay);
 939
 940         return cdata;
 941 }
 942
 943 static int tegra_qspi_setup(struct spi_device *spi)
 944 {
 945         struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master);
 946         struct tegra_qspi_client_data *cdata = spi->controller_data;
 947         unsigned long flags;
 948         u32 val;
 949         int ret;
 950
 951         ret = pm_runtime_resume_and_get(tqspi->dev);
 952         if (ret < 0) {
 953                 dev_err(tqspi->dev, "failed to get runtime PM: %d\n", ret);
 954                 return ret;
 955         }
 956
 957         if (!cdata) {
 958                 cdata = tegra_qspi_parse_cdata_dt(spi);
 959                 spi->controller_data = cdata;
 960         }
 961         spin_lock_irqsave(&tqspi->lock, flags);
 962
 963         /* keep default cs state to inactive */
 964         val = tqspi->def_command1_reg;
 965         val |= QSPI_CS_SEL(spi_get_chipselect(spi, 0));
 966         if (spi->mode & SPI_CS_HIGH)
 967                 val &= ~QSPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 968         else
 969                 val |= QSPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 970
 971         tqspi->def_command1_reg = val;
 972         tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1);
 973
 974         spin_unlock_irqrestore(&tqspi->lock, flags);
 975
 976         pm_runtime_put(tqspi->dev);
 977
 978         return 0;
 979 }
 980
 981 static void tegra_qspi_dump_regs(struct tegra_qspi *tqspi)
 982 {
 983         dev_dbg(tqspi->dev, "============ QSPI REGISTER DUMP ============\n");
 984         dev_dbg(tqspi->dev, "Command1:    0x%08x | Command2:    0x%08x\n",
 985                 tegra_qspi_readl(tqspi, QSPI_COMMAND1),
 986                 tegra_qspi_readl(tqspi, QSPI_COMMAND2));
 987         dev_dbg(tqspi->dev, "DMA_CTL:     0x%08x | DMA_BLK:     0x%08x\n",
 988                 tegra_qspi_readl(tqspi, QSPI_DMA_CTL),
 989                 tegra_qspi_readl(tqspi, QSPI_DMA_BLK));
 990         dev_dbg(tqspi->dev, "INTR_MASK:  0x%08x | MISC: 0x%08x\n",
 991                 tegra_qspi_readl(tqspi, QSPI_INTR_MASK),
 992                 tegra_qspi_readl(tqspi, QSPI_MISC_REG));
 993         dev_dbg(tqspi->dev, "TRANS_STAT:  0x%08x | FIFO_STATUS: 0x%08x\n",
 994                 tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS),
 995                 tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS));
 996 }
 997
 998 static void tegra_qspi_handle_error(struct tegra_qspi *tqspi)
 999 {
1000         dev_err(tqspi->dev, "error in transfer, fifo status 0x%08x\n", tqspi->status_reg);
1001         tegra_qspi_dump_regs(tqspi);
1002         tegra_qspi_flush_fifos(tqspi, true);
1003         if (device_reset(tqspi->dev) < 0)
1004                 dev_warn_once(tqspi->dev, "device reset failed\n");
1005 }
1006
1007 static void tegra_qspi_transfer_end(struct spi_device *spi)
1008 {
1009         struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master);
1010         int cs_val = (spi->mode & SPI_CS_HIGH) ? 0 : 1;
1011
1012         if (cs_val)
1013                 tqspi->command1_reg |= QSPI_CS_SW_VAL;
1014         else
1015                 tqspi->command1_reg &= ~QSPI_CS_SW_VAL;
1016         tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1);
1017         tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1);
1018 }
1019
1020 static u32 tegra_qspi_cmd_config(bool is_ddr, u8 bus_width, u8 len)
1021 {
1022         u32 cmd_config = 0;
1023
1024         /* Extract Command configuration and value */
1025         if (is_ddr)
1026                 cmd_config |= QSPI_COMMAND_SDR_DDR;
1027         else
1028                 cmd_config &= ~QSPI_COMMAND_SDR_DDR;
1029
1030         cmd_config |= QSPI_COMMAND_X1_X2_X4(bus_width);
1031         cmd_config |= QSPI_COMMAND_SIZE_SET((len * 8) - 1);
1032
1033         return cmd_config;
1034 }
1035
1036 static u32 tegra_qspi_addr_config(bool is_ddr, u8 bus_width, u8 len)
1037 {
1038         u32 addr_config = 0;
1039
1040         /* Extract Address configuration and value */
1041         is_ddr = 0; //Only SDR mode supported
1042         bus_width = 0; //X1 mode
1043
1044         if (is_ddr)
1045                 addr_config |= QSPI_ADDRESS_SDR_DDR;
1046         else
1047                 addr_config &= ~QSPI_ADDRESS_SDR_DDR;
1048
1049         addr_config |= QSPI_ADDRESS_X1_X2_X4(bus_width);
1050         addr_config |= QSPI_ADDRESS_SIZE_SET((len * 8) - 1);
1051
1052         return addr_config;
1053 }
1054
1055 static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi,
1056                                         struct spi_message *msg)
1057 {
1058         bool is_first_msg = true;
1059         struct spi_transfer *xfer;
1060         struct spi_device *spi = msg->spi;
1061         u8 transfer_phase = 0;
1062         u32 cmd1 = 0, dma_ctl = 0;
1063         int ret = 0;
1064         u32 address_value = 0;
1065         u32 cmd_config = 0, addr_config = 0;
1066         u8 cmd_value = 0, val = 0;
1067
1068         /* Enable Combined sequence mode */
1069         val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG);
1070         if (spi->mode & SPI_TPM_HW_FLOW) {
1071                 if (tqspi->soc_data->supports_tpm)
1072                         val |= QSPI_TPM_WAIT_POLL_EN;
1073                 else
1074                         return -EIO;
1075         }
1076         val |= QSPI_CMB_SEQ_EN;
1077         tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG);
1078         /* Process individual transfer list */
1079         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1080                 switch (transfer_phase) {
1081                 case CMD_TRANSFER:
1082                         /* X1 SDR mode */
1083                         cmd_config = tegra_qspi_cmd_config(false, 0,
1084                                                            xfer->len);
1085                         cmd_value = *((const u8 *)(xfer->tx_buf));
1086                         break;
1087                 case ADDR_TRANSFER:
1088                         /* X1 SDR mode */
1089                         addr_config = tegra_qspi_addr_config(false, 0,
1090                                                              xfer->len);
1091                         address_value = *((const u32 *)(xfer->tx_buf));
1092                         break;
1093                 case DATA_TRANSFER:
1094                         /* Program Command, Address value in register */
1095                         tegra_qspi_writel(tqspi, cmd_value, QSPI_CMB_SEQ_CMD);
1096                         tegra_qspi_writel(tqspi, address_value,
1097                                           QSPI_CMB_SEQ_ADDR);
1098                         /* Program Command and Address config in register */
1099                         tegra_qspi_writel(tqspi, cmd_config,
1100                                           QSPI_CMB_SEQ_CMD_CFG);
1101                         tegra_qspi_writel(tqspi, addr_config,
1102                                           QSPI_CMB_SEQ_ADDR_CFG);
1103
1104                         reinit_completion(&tqspi->xfer_completion);
1105                         cmd1 = tegra_qspi_setup_transfer_one(spi, xfer,
1106                                                              is_first_msg);
1107                         ret = tegra_qspi_start_transfer_one(spi, xfer,
1108                                                             cmd1);
1109
1110                         if (ret < 0) {
1111                                 dev_err(tqspi->dev, "Failed to start transfer-one: %d\n",
1112                                         ret);
1113                                 return ret;
1114                         }
1115
1116                         is_first_msg = false;
1117                         ret = wait_for_completion_timeout
1118                                         (&tqspi->xfer_completion,
1119                                         QSPI_DMA_TIMEOUT);
1120
1121                         if (WARN_ON(ret == 0)) {
1122                                 dev_err(tqspi->dev, "QSPI Transfer failed with timeout: %d\n",
1123                                         ret);
1124                                 if (tqspi->is_curr_dma_xfer &&
1125                                     (tqspi->cur_direction & DATA_DIR_TX))
1126                                         dmaengine_terminate_all
1127                                                 (tqspi->tx_dma_chan);
1128
1129                                 if (tqspi->is_curr_dma_xfer &&
1130                                     (tqspi->cur_direction & DATA_DIR_RX))
1131                                         dmaengine_terminate_all
1132                                                 (tqspi->rx_dma_chan);
1133
1134                                 /* Abort transfer by resetting pio/dma bit */
1135                                 if (!tqspi->is_curr_dma_xfer) {
1136                                         cmd1 = tegra_qspi_readl
1137                                                         (tqspi,
1138                                                          QSPI_COMMAND1);
1139                                         cmd1 &= ~QSPI_PIO;
1140                                         tegra_qspi_writel
1141                                                         (tqspi, cmd1,
1142                                                          QSPI_COMMAND1);
1143                                 } else {
1144                                         dma_ctl = tegra_qspi_readl
1145                                                         (tqspi,
1146                                                          QSPI_DMA_CTL);
1147                                         dma_ctl &= ~QSPI_DMA_EN;
1148                                         tegra_qspi_writel(tqspi, dma_ctl,
1149                                                           QSPI_DMA_CTL);
1150                                 }
1151
1152                                 /* Reset controller if timeout happens */
1153                                 if (device_reset(tqspi->dev) < 0)
1154                                         dev_warn_once(tqspi->dev,
1155                                                       "device reset failed\n");
1156                                 ret = -EIO;
1157                                 goto exit;
1158                         }
1159
1160                         if (tqspi->tx_status ||  tqspi->rx_status) {
1161                                 dev_err(tqspi->dev, "QSPI Transfer failed\n");
1162                                 tqspi->tx_status = 0;
1163                                 tqspi->rx_status = 0;
1164                                 ret = -EIO;
1165                                 goto exit;
1166                         }
1167                         if (!xfer->cs_change) {
1168                                 tegra_qspi_transfer_end(spi);
1169                                 spi_transfer_delay_exec(xfer);
1170                         }
1171                         break;
1172                 default:
1173                         ret = -EINVAL;
1174                         goto exit;
1175                 }
1176                 msg->actual_length += xfer->len;
1177                 transfer_phase++;
1178         }
1179         ret = 0;
1180
1181 exit:
1182         msg->status = ret;
1183         if (ret < 0) {
1184                 tegra_qspi_transfer_end(spi);
1185                 spi_transfer_delay_exec(xfer);
1186         }
1187
1188         return ret;
1189 }
1190
1191 static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi,
1192                                             struct spi_message *msg)
1193 {
1194         struct spi_device *spi = msg->spi;
1195         struct spi_transfer *transfer;
1196         bool is_first_msg = true;
1197         int ret = 0, val = 0;
1198
1199         msg->status = 0;
1200         msg->actual_length = 0;
1201         tqspi->tx_status = 0;
1202         tqspi->rx_status = 0;
1203
1204         /* Disable Combined sequence mode */
1205         val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG);
1206         val &= ~QSPI_CMB_SEQ_EN;
1207         if (tqspi->soc_data->supports_tpm)
1208                 val &= ~QSPI_TPM_WAIT_POLL_EN;
1209         tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG);
1210         list_for_each_entry(transfer, &msg->transfers, transfer_list) {
1211                 struct spi_transfer *xfer = transfer;
1212                 u8 dummy_bytes = 0;
1213                 u32 cmd1;
1214
1215                 tqspi->dummy_cycles = 0;
1216                 /*
1217                  * Tegra QSPI hardware supports dummy bytes transfer after actual transfer
1218                  * bytes based on programmed dummy clock cycles in the QSPI_MISC register.
1219                  * So, check if the next transfer is dummy data transfer and program dummy
1220                  * clock cycles along with the current transfer and skip next transfer.
1221                  */
1222                 if (!list_is_last(&xfer->transfer_list, &msg->transfers)) {
1223                         struct spi_transfer *next_xfer;
1224
1225                         next_xfer = list_next_entry(xfer, transfer_list);
1226                         if (next_xfer->dummy_data) {
1227                                 u32 dummy_cycles = next_xfer->len * 8 / next_xfer->tx_nbits;
1228
1229                                 if (dummy_cycles <= QSPI_DUMMY_CYCLES_MAX) {
1230                                         tqspi->dummy_cycles = dummy_cycles;
1231                                         dummy_bytes = next_xfer->len;
1232                                         transfer = next_xfer;
1233                                 }
1234                         }
1235                 }
1236
1237                 reinit_completion(&tqspi->xfer_completion);
1238
1239                 cmd1 = tegra_qspi_setup_transfer_one(spi, xfer, is_first_msg);
1240
1241                 ret = tegra_qspi_start_transfer_one(spi, xfer, cmd1);
1242                 if (ret < 0) {
1243                         dev_err(tqspi->dev, "failed to start transfer: %d\n", ret);
1244                         goto complete_xfer;
1245                 }
1246
1247                 ret = wait_for_completion_timeout(&tqspi->xfer_completion,
1248                                                   QSPI_DMA_TIMEOUT);
1249                 if (WARN_ON(ret == 0)) {
1250                         dev_err(tqspi->dev, "transfer timeout\n");
1251                         if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_TX))
1252                                 dmaengine_terminate_all(tqspi->tx_dma_chan);
1253                         if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_RX))
1254                                 dmaengine_terminate_all(tqspi->rx_dma_chan);
1255                         tegra_qspi_handle_error(tqspi);
1256                         ret = -EIO;
1257                         goto complete_xfer;
1258                 }
1259
1260                 if (tqspi->tx_status ||  tqspi->rx_status) {
1261                         tegra_qspi_handle_error(tqspi);
1262                         ret = -EIO;
1263                         goto complete_xfer;
1264                 }
1265
1266                 msg->actual_length += xfer->len + dummy_bytes;
1267
1268 complete_xfer:
1269                 if (ret < 0) {
1270                         tegra_qspi_transfer_end(spi);
1271                         spi_transfer_delay_exec(xfer);
1272                         goto exit;
1273                 }
1274
1275                 if (list_is_last(&xfer->transfer_list, &msg->transfers)) {
1276                         /* de-activate CS after last transfer only when cs_change is not set */
1277                         if (!xfer->cs_change) {
1278                                 tegra_qspi_transfer_end(spi);
1279                                 spi_transfer_delay_exec(xfer);
1280                         }
1281                 } else if (xfer->cs_change) {
1282                          /* de-activated CS between the transfers only when cs_change is set */
1283                         tegra_qspi_transfer_end(spi);
1284                         spi_transfer_delay_exec(xfer);
1285                 }
1286         }
1287
1288         ret = 0;
1289 exit:
1290         msg->status = ret;
1291
1292         return ret;
1293 }
1294
1295 static bool tegra_qspi_validate_cmb_seq(struct tegra_qspi *tqspi,
1296                                         struct spi_message *msg)
1297 {
1298         int transfer_count = 0;
1299         struct spi_transfer *xfer;
1300
1301         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1302                 transfer_count++;
1303         }
1304         if (!tqspi->soc_data->cmb_xfer_capable || transfer_count != 3)
1305                 return false;
1306         xfer = list_first_entry(&msg->transfers, typeof(*xfer),
1307                                 transfer_list);
1308         if (xfer->len > 2)
1309                 return false;
1310         xfer = list_next_entry(xfer, transfer_list);
1311         if (xfer->len > 4 || xfer->len < 3)
1312                 return false;
1313         xfer = list_next_entry(xfer, transfer_list);
1314         if (!tqspi->soc_data->has_dma && xfer->len > (QSPI_FIFO_DEPTH << 2))
1315                 return false;
1316
1317         return true;
1318 }
1319
1320 static int tegra_qspi_transfer_one_message(struct spi_master *master,
1321                                            struct spi_message *msg)
1322 {
1323         struct tegra_qspi *tqspi = spi_master_get_devdata(master);
1324         int ret;
1325
1326         if (tegra_qspi_validate_cmb_seq(tqspi, msg))
1327                 ret = tegra_qspi_combined_seq_xfer(tqspi, msg);
1328         else
1329                 ret = tegra_qspi_non_combined_seq_xfer(tqspi, msg);
1330
1331         spi_finalize_current_message(master);
1332
1333         return ret;
1334 }
1335
1336 static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi *tqspi)
1337 {
1338         struct spi_transfer *t = tqspi->curr_xfer;
1339         unsigned long flags;
1340
1341         spin_lock_irqsave(&tqspi->lock, flags);
1342
1343         if (tqspi->tx_status ||  tqspi->rx_status) {
1344                 tegra_qspi_handle_error(tqspi);
1345                 complete(&tqspi->xfer_completion);
1346                 goto exit;
1347         }
1348
1349         if (tqspi->cur_direction & DATA_DIR_RX)
1350                 tegra_qspi_read_rx_fifo_to_client_rxbuf(tqspi, t);
1351
1352         if (tqspi->cur_direction & DATA_DIR_TX)
1353                 tqspi->cur_pos = tqspi->cur_tx_pos;
1354         else
1355                 tqspi->cur_pos = tqspi->cur_rx_pos;
1356
1357         if (tqspi->cur_pos == t->len) {
1358                 complete(&tqspi->xfer_completion);
1359                 goto exit;
1360         }
1361
1362         tegra_qspi_calculate_curr_xfer_param(tqspi, t);
1363         tegra_qspi_start_cpu_based_transfer(tqspi, t);
1364 exit:
1365         spin_unlock_irqrestore(&tqspi->lock, flags);
1366         return IRQ_HANDLED;
1367 }
1368
1369 static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi)
1370 {
1371         struct spi_transfer *t = tqspi->curr_xfer;
1372         unsigned int total_fifo_words;
1373         unsigned long flags;
1374         long wait_status;
1375         int err = 0;
1376
1377         if (tqspi->cur_direction & DATA_DIR_TX) {
1378                 if (tqspi->tx_status) {
1379                         dmaengine_terminate_all(tqspi->tx_dma_chan);
1380                         err += 1;
1381                 } else {
1382                         wait_status = wait_for_completion_interruptible_timeout(
1383                                 &tqspi->tx_dma_complete, QSPI_DMA_TIMEOUT);
1384                         if (wait_status <= 0) {
1385                                 dmaengine_terminate_all(tqspi->tx_dma_chan);
1386                                 dev_err(tqspi->dev, "failed TX DMA transfer\n");
1387                                 err += 1;
1388                         }
1389                 }
1390         }
1391
1392         if (tqspi->cur_direction & DATA_DIR_RX) {
1393                 if (tqspi->rx_status) {
1394                         dmaengine_terminate_all(tqspi->rx_dma_chan);
1395                         err += 2;
1396                 } else {
1397                         wait_status = wait_for_completion_interruptible_timeout(
1398                                 &tqspi->rx_dma_complete, QSPI_DMA_TIMEOUT);
1399                         if (wait_status <= 0) {
1400                                 dmaengine_terminate_all(tqspi->rx_dma_chan);
1401                                 dev_err(tqspi->dev, "failed RX DMA transfer\n");
1402                                 err += 2;
1403                         }
1404                 }
1405         }
1406
1407         spin_lock_irqsave(&tqspi->lock, flags);
1408
1409         if (err) {
1410                 tegra_qspi_dma_unmap_xfer(tqspi, t);
1411                 tegra_qspi_handle_error(tqspi);
1412                 complete(&tqspi->xfer_completion);
1413                 goto exit;
1414         }
1415
1416         if (tqspi->cur_direction & DATA_DIR_RX)
1417                 tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf(tqspi, t);
1418
1419         if (tqspi->cur_direction & DATA_DIR_TX)
1420                 tqspi->cur_pos = tqspi->cur_tx_pos;
1421         else
1422                 tqspi->cur_pos = tqspi->cur_rx_pos;
1423
1424         if (tqspi->cur_pos == t->len) {
1425                 tegra_qspi_dma_unmap_xfer(tqspi, t);
1426                 complete(&tqspi->xfer_completion);
1427                 goto exit;
1428         }
1429
1430         tegra_qspi_dma_unmap_xfer(tqspi, t);
1431
1432         /* continue transfer in current message */
1433         total_fifo_words = tegra_qspi_calculate_curr_xfer_param(tqspi, t);
1434         if (total_fifo_words > QSPI_FIFO_DEPTH)
1435                 err = tegra_qspi_start_dma_based_transfer(tqspi, t);
1436         else
1437                 err = tegra_qspi_start_cpu_based_transfer(tqspi, t);
1438
1439 exit:
1440         spin_unlock_irqrestore(&tqspi->lock, flags);
1441         return IRQ_HANDLED;
1442 }
1443
1444 static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data)
1445 {
1446         struct tegra_qspi *tqspi = context_data;
1447
1448         tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
1449
1450         if (tqspi->cur_direction & DATA_DIR_TX)
1451                 tqspi->tx_status = tqspi->status_reg & (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF);
1452
1453         if (tqspi->cur_direction & DATA_DIR_RX)
1454                 tqspi->rx_status = tqspi->status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF);
1455
1456         tegra_qspi_mask_clear_irq(tqspi);
1457
1458         if (!tqspi->is_curr_dma_xfer)
1459                 return handle_cpu_based_xfer(tqspi);
1460
1461         return handle_dma_based_xfer(tqspi);
1462 }
1463
1464 static struct tegra_qspi_soc_data tegra210_qspi_soc_data = {
1465         .has_dma = true,
1466         .cmb_xfer_capable = false,
1467         .supports_tpm = false,
1468         .cs_count = 1,
1469 };
1470
1471 static struct tegra_qspi_soc_data tegra186_qspi_soc_data = {
1472         .has_dma = true,
1473         .cmb_xfer_capable = true,
1474         .supports_tpm = false,
1475         .cs_count = 1,
1476 };
1477
1478 static struct tegra_qspi_soc_data tegra234_qspi_soc_data = {
1479         .has_dma = false,
1480         .cmb_xfer_capable = true,
1481         .supports_tpm = true,
1482         .cs_count = 1,
1483 };
1484
1485 static struct tegra_qspi_soc_data tegra241_qspi_soc_data = {
1486         .has_dma = false,
1487         .cmb_xfer_capable = true,
1488         .supports_tpm = true,
1489         .cs_count = 4,
1490 };
1491
1492 static const struct of_device_id tegra_qspi_of_match[] = {
1493         {
1494                 .compatible = "nvidia,tegra210-qspi",
1495                 .data       = &tegra210_qspi_soc_data,
1496         }, {
1497                 .compatible = "nvidia,tegra186-qspi",
1498                 .data       = &tegra186_qspi_soc_data,
1499         }, {
1500                 .compatible = "nvidia,tegra194-qspi",
1501                 .data       = &tegra186_qspi_soc_data,
1502         }, {
1503                 .compatible = "nvidia,tegra234-qspi",
1504                 .data       = &tegra234_qspi_soc_data,
1505         }, {
1506                 .compatible = "nvidia,tegra241-qspi",
1507                 .data       = &tegra241_qspi_soc_data,
1508         },
1509         {}
1510 };
1511
1512 MODULE_DEVICE_TABLE(of, tegra_qspi_of_match);
1513
1514 #ifdef CONFIG_ACPI
1515 static const struct acpi_device_id tegra_qspi_acpi_match[] = {
1516         {
1517                 .id = "NVDA1213",
1518                 .driver_data = (kernel_ulong_t)&tegra210_qspi_soc_data,
1519         }, {
1520                 .id = "NVDA1313",
1521                 .driver_data = (kernel_ulong_t)&tegra186_qspi_soc_data,
1522         }, {
1523                 .id = "NVDA1413",
1524                 .driver_data = (kernel_ulong_t)&tegra234_qspi_soc_data,
1525         }, {
1526                 .id = "NVDA1513",
1527                 .driver_data = (kernel_ulong_t)&tegra241_qspi_soc_data,
1528         },
1529         {}
1530 };
1531
1532 MODULE_DEVICE_TABLE(acpi, tegra_qspi_acpi_match);
1533 #endif
1534
1535 static int tegra_qspi_probe(struct platform_device *pdev)
1536 {
1537         struct spi_master       *master;
1538         struct tegra_qspi       *tqspi;
1539         struct resource         *r;
1540         int ret, qspi_irq;
1541         int bus_num;
1542
1543         master = devm_spi_alloc_master(&pdev->dev, sizeof(*tqspi));
1544         if (!master)
1545                 return -ENOMEM;
1546
1547         platform_set_drvdata(pdev, master);
1548         tqspi = spi_master_get_devdata(master);
1549
1550         master->mode_bits = SPI_MODE_0 | SPI_MODE_3 | SPI_CS_HIGH |
1551                             SPI_TX_DUAL | SPI_RX_DUAL | SPI_TX_QUAD | SPI_RX_QUAD;
1552         master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(16) | SPI_BPW_MASK(8);
1553         master->flags = SPI_CONTROLLER_HALF_DUPLEX;
1554         master->setup = tegra_qspi_setup;
1555         master->transfer_one_message = tegra_qspi_transfer_one_message;
1556         master->num_chipselect = 1;
1557         master->auto_runtime_pm = true;
1558
1559         bus_num = of_alias_get_id(pdev->dev.of_node, "spi");
1560         if (bus_num >= 0)
1561                 master->bus_num = bus_num;
1562
1563         tqspi->master = master;
1564         tqspi->dev = &pdev->dev;
1565         spin_lock_init(&tqspi->lock);
1566
1567         tqspi->soc_data = device_get_match_data(&pdev->dev);
1568         master->num_chipselect = tqspi->soc_data->cs_count;
1569         tqspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
1570         if (IS_ERR(tqspi->base))
1571                 return PTR_ERR(tqspi->base);
1572
1573         tqspi->phys = r->start;
1574         qspi_irq = platform_get_irq(pdev, 0);
1575         if (qspi_irq < 0)
1576                 return qspi_irq;
1577         tqspi->irq = qspi_irq;
1578
1579         if (!has_acpi_companion(tqspi->dev)) {
1580                 tqspi->clk = devm_clk_get(&pdev->dev, "qspi");
1581                 if (IS_ERR(tqspi->clk)) {
1582                         ret = PTR_ERR(tqspi->clk);
1583                         dev_err(&pdev->dev, "failed to get clock: %d\n", ret);
1584                         return ret;
1585                 }
1586
1587         }
1588
1589         tqspi->max_buf_size = QSPI_FIFO_DEPTH << 2;
1590         tqspi->dma_buf_size = DEFAULT_QSPI_DMA_BUF_LEN;
1591
1592         ret = tegra_qspi_init_dma(tqspi);
1593         if (ret < 0)
1594                 return ret;
1595
1596         if (tqspi->use_dma)
1597                 tqspi->max_buf_size = tqspi->dma_buf_size;
1598
1599         init_completion(&tqspi->tx_dma_complete);
1600         init_completion(&tqspi->rx_dma_complete);
1601         init_completion(&tqspi->xfer_completion);
1602
1603         pm_runtime_enable(&pdev->dev);
1604         ret = pm_runtime_resume_and_get(&pdev->dev);
1605         if (ret < 0) {
1606                 dev_err(&pdev->dev, "failed to get runtime PM: %d\n", ret);
1607                 goto exit_pm_disable;
1608         }
1609
1610         if (device_reset(tqspi->dev) < 0)
1611                 dev_warn_once(tqspi->dev, "device reset failed\n");
1612
1613         tqspi->def_command1_reg = QSPI_M_S | QSPI_CS_SW_HW |  QSPI_CS_SW_VAL;
1614         tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1);
1615         tqspi->spi_cs_timing1 = tegra_qspi_readl(tqspi, QSPI_CS_TIMING1);
1616         tqspi->spi_cs_timing2 = tegra_qspi_readl(tqspi, QSPI_CS_TIMING2);
1617         tqspi->def_command2_reg = tegra_qspi_readl(tqspi, QSPI_COMMAND2);
1618
1619         pm_runtime_put(&pdev->dev);
1620
1621         ret = request_threaded_irq(tqspi->irq, NULL,
1622                                    tegra_qspi_isr_thread, IRQF_ONESHOT,
1623                                    dev_name(&pdev->dev), tqspi);
1624         if (ret < 0) {
1625                 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", tqspi->irq, ret);
1626                 goto exit_pm_disable;
1627         }
1628
1629         master->dev.of_node = pdev->dev.of_node;
1630         ret = spi_register_master(master);
1631         if (ret < 0) {
1632                 dev_err(&pdev->dev, "failed to register master: %d\n", ret);
1633                 goto exit_free_irq;
1634         }
1635
1636         return 0;
1637
1638 exit_free_irq:
1639         free_irq(qspi_irq, tqspi);
1640 exit_pm_disable:
1641         pm_runtime_force_suspend(&pdev->dev);
1642         tegra_qspi_deinit_dma(tqspi);
1643         return ret;
1644 }
1645
1646 static void tegra_qspi_remove(struct platform_device *pdev)
1647 {
1648         struct spi_master *master = platform_get_drvdata(pdev);
1649         struct tegra_qspi *tqspi = spi_master_get_devdata(master);
1650
1651         spi_unregister_master(master);
1652         free_irq(tqspi->irq, tqspi);
1653         pm_runtime_force_suspend(&pdev->dev);
1654         tegra_qspi_deinit_dma(tqspi);
1655 }
1656
1657 static int __maybe_unused tegra_qspi_suspend(struct device *dev)
1658 {
1659         struct spi_master *master = dev_get_drvdata(dev);
1660
1661         return spi_master_suspend(master);
1662 }
1663
1664 static int __maybe_unused tegra_qspi_resume(struct device *dev)
1665 {
1666         struct spi_master *master = dev_get_drvdata(dev);
1667         struct tegra_qspi *tqspi = spi_master_get_devdata(master);
1668         int ret;
1669
1670         ret = pm_runtime_resume_and_get(dev);
1671         if (ret < 0) {
1672                 dev_err(dev, "failed to get runtime PM: %d\n", ret);
1673                 return ret;
1674         }
1675
1676         tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1);
1677         tegra_qspi_writel(tqspi, tqspi->def_command2_reg, QSPI_COMMAND2);
1678         pm_runtime_put(dev);
1679
1680         return spi_master_resume(master);
1681 }
1682
1683 static int __maybe_unused tegra_qspi_runtime_suspend(struct device *dev)
1684 {
1685         struct spi_master *master = dev_get_drvdata(dev);
1686         struct tegra_qspi *tqspi = spi_master_get_devdata(master);
1687
1688         /* Runtime pm disabled with ACPI */
1689         if (has_acpi_companion(tqspi->dev))
1690                 return 0;
1691         /* flush all write which are in PPSB queue by reading back */
1692         tegra_qspi_readl(tqspi, QSPI_COMMAND1);
1693
1694         clk_disable_unprepare(tqspi->clk);
1695
1696         return 0;
1697 }
1698
1699 static int __maybe_unused tegra_qspi_runtime_resume(struct device *dev)
1700 {
1701         struct spi_master *master = dev_get_drvdata(dev);
1702         struct tegra_qspi *tqspi = spi_master_get_devdata(master);
1703         int ret;
1704
1705         /* Runtime pm disabled with ACPI */
1706         if (has_acpi_companion(tqspi->dev))
1707                 return 0;
1708         ret = clk_prepare_enable(tqspi->clk);
1709         if (ret < 0)
1710                 dev_err(tqspi->dev, "failed to enable clock: %d\n", ret);
1711
1712         return ret;
1713 }
1714
1715 static const struct dev_pm_ops tegra_qspi_pm_ops = {
1716         SET_RUNTIME_PM_OPS(tegra_qspi_runtime_suspend, tegra_qspi_runtime_resume, NULL)
1717         SET_SYSTEM_SLEEP_PM_OPS(tegra_qspi_suspend, tegra_qspi_resume)
1718 };
1719
1720 static struct platform_driver tegra_qspi_driver = {
1721         .driver = {
1722                 .name           = "tegra-qspi",
1723                 .pm             = &tegra_qspi_pm_ops,
1724                 .of_match_table = tegra_qspi_of_match,
1725                 .acpi_match_table = ACPI_PTR(tegra_qspi_acpi_match),
1726         },
1727         .probe =        tegra_qspi_probe,
1728         .remove_new =   tegra_qspi_remove,
1729 };
1730 module_platform_driver(tegra_qspi_driver);
1731
1732 MODULE_ALIAS("platform:qspi-tegra");
1733 MODULE_DESCRIPTION("NVIDIA Tegra QSPI Controller Driver");
1734 MODULE_AUTHOR("Sowjanya Komatineni <skomatineni@nvidia.com>");
1735 MODULE_LICENSE("GPL v2");