drivers/spi/spi-qcom-qspi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   3
   4 #include <linux/clk.h>
   5 #include <linux/interconnect.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/of_platform.h>
  11 #include <linux/pm_runtime.h>
  12 #include <linux/pm_opp.h>
  13 #include <linux/spi/spi.h>
  14 #include <linux/spi/spi-mem.h>
  15
  16
  17 #define QSPI_NUM_CS             2
  18 #define QSPI_BYTES_PER_WORD     4
  19
  20 #define MSTR_CONFIG             0x0000
  21 #define FULL_CYCLE_MODE         BIT(3)
  22 #define FB_CLK_EN               BIT(4)
  23 #define PIN_HOLDN               BIT(6)
  24 #define PIN_WPN                 BIT(7)
  25 #define DMA_ENABLE              BIT(8)
  26 #define BIG_ENDIAN_MODE         BIT(9)
  27 #define SPI_MODE_MSK            0xc00
  28 #define SPI_MODE_SHFT           10
  29 #define CHIP_SELECT_NUM         BIT(12)
  30 #define SBL_EN                  BIT(13)
  31 #define LPA_BASE_MSK            0x3c000
  32 #define LPA_BASE_SHFT           14
  33 #define TX_DATA_DELAY_MSK       0xc0000
  34 #define TX_DATA_DELAY_SHFT      18
  35 #define TX_CLK_DELAY_MSK        0x300000
  36 #define TX_CLK_DELAY_SHFT       20
  37 #define TX_CS_N_DELAY_MSK       0xc00000
  38 #define TX_CS_N_DELAY_SHFT      22
  39 #define TX_DATA_OE_DELAY_MSK    0x3000000
  40 #define TX_DATA_OE_DELAY_SHFT   24
  41
  42 #define AHB_MASTER_CFG                          0x0004
  43 #define HMEM_TYPE_START_MID_TRANS_MSK           0x7
  44 #define HMEM_TYPE_START_MID_TRANS_SHFT          0
  45 #define HMEM_TYPE_LAST_TRANS_MSK                0x38
  46 #define HMEM_TYPE_LAST_TRANS_SHFT               3
  47 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK  0xc0
  48 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
  49 #define HMEMTYPE_READ_TRANS_MSK                 0x700
  50 #define HMEMTYPE_READ_TRANS_SHFT                8
  51 #define HSHARED                                 BIT(11)
  52 #define HINNERSHARED                            BIT(12)
  53
  54 #define MSTR_INT_EN             0x000C
  55 #define MSTR_INT_STATUS         0x0010
  56 #define RESP_FIFO_UNDERRUN      BIT(0)
  57 #define RESP_FIFO_NOT_EMPTY     BIT(1)
  58 #define RESP_FIFO_RDY           BIT(2)
  59 #define HRESP_FROM_NOC_ERR      BIT(3)
  60 #define WR_FIFO_EMPTY           BIT(9)
  61 #define WR_FIFO_FULL            BIT(10)
  62 #define WR_FIFO_OVERRUN         BIT(11)
  63 #define TRANSACTION_DONE        BIT(16)
  64 #define QSPI_ERR_IRQS           (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
  65                                  WR_FIFO_OVERRUN)
  66 #define QSPI_ALL_IRQS           (QSPI_ERR_IRQS | RESP_FIFO_RDY | \
  67                                  WR_FIFO_EMPTY | WR_FIFO_FULL | \
  68                                  TRANSACTION_DONE)
  69
  70 #define PIO_XFER_CTRL           0x0014
  71 #define REQUEST_COUNT_MSK       0xffff
  72
  73 #define PIO_XFER_CFG            0x0018
  74 #define TRANSFER_DIRECTION      BIT(0)
  75 #define MULTI_IO_MODE_MSK       0xe
  76 #define MULTI_IO_MODE_SHFT      1
  77 #define TRANSFER_FRAGMENT       BIT(8)
  78 #define SDR_1BIT                1
  79 #define SDR_2BIT                2
  80 #define SDR_4BIT                3
  81 #define DDR_1BIT                5
  82 #define DDR_2BIT                6
  83 #define DDR_4BIT                7
  84 #define DMA_DESC_SINGLE_SPI     1
  85 #define DMA_DESC_DUAL_SPI       2
  86 #define DMA_DESC_QUAD_SPI       3
  87
  88 #define PIO_XFER_STATUS         0x001c
  89 #define WR_FIFO_BYTES_MSK       0xffff0000
  90 #define WR_FIFO_BYTES_SHFT      16
  91
  92 #define PIO_DATAOUT_1B          0x0020
  93 #define PIO_DATAOUT_4B          0x0024
  94
  95 #define RD_FIFO_CFG             0x0028
  96 #define CONTINUOUS_MODE         BIT(0)
  97
  98 #define RD_FIFO_STATUS  0x002c
  99 #define FIFO_EMPTY      BIT(11)
 100 #define WR_CNTS_MSK     0x7f0
 101 #define WR_CNTS_SHFT    4
 102 #define RDY_64BYTE      BIT(3)
 103 #define RDY_32BYTE      BIT(2)
 104 #define RDY_16BYTE      BIT(1)
 105 #define FIFO_RDY        BIT(0)
 106
 107 #define RD_FIFO_RESET           0x0030
 108 #define RESET_FIFO              BIT(0)
 109
 110 #define CUR_MEM_ADDR            0x0048
 111 #define HW_VERSION              0x004c
 112 #define RD_FIFO                 0x0050
 113 #define SAMPLING_CLK_CFG        0x0090
 114 #define SAMPLING_CLK_STATUS     0x0094
 115
 116
 117 enum qspi_dir {
 118         QSPI_READ,
 119         QSPI_WRITE,
 120 };
 121
 122 struct qspi_xfer {
 123         union {
 124                 const void *tx_buf;
 125                 void *rx_buf;
 126         };
 127         unsigned int rem_bytes;
 128         unsigned int buswidth;
 129         enum qspi_dir dir;
 130         bool is_last;
 131 };
 132
 133 enum qspi_clocks {
 134         QSPI_CLK_CORE,
 135         QSPI_CLK_IFACE,
 136         QSPI_NUM_CLKS
 137 };
 138
 139 struct qcom_qspi {
 140         void __iomem *base;
 141         struct device *dev;
 142         struct clk_bulk_data *clks;
 143         struct qspi_xfer xfer;
 144         struct icc_path *icc_path_cpu_to_qspi;
 145         unsigned long last_speed;
 146         /* Lock to protect data accessed by IRQs */
 147         spinlock_t lock;
 148 };
 149
 150 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
 151                                    unsigned int buswidth)
 152 {
 153         switch (buswidth) {
 154         case 1:
 155                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 156         case 2:
 157                 return SDR_2BIT << MULTI_IO_MODE_SHFT;
 158         case 4:
 159                 return SDR_4BIT << MULTI_IO_MODE_SHFT;
 160         default:
 161                 dev_warn_once(ctrl->dev,
 162                                 "Unexpected bus width: %u\n", buswidth);
 163                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 164         }
 165 }
 166
 167 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
 168 {
 169         u32 pio_xfer_cfg;
 170         const struct qspi_xfer *xfer;
 171
 172         xfer = &ctrl->xfer;
 173         pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
 174         pio_xfer_cfg &= ~TRANSFER_DIRECTION;
 175         pio_xfer_cfg |= xfer->dir;
 176         if (xfer->is_last)
 177                 pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
 178         else
 179                 pio_xfer_cfg |= TRANSFER_FRAGMENT;
 180         pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
 181         pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
 182
 183         writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
 184 }
 185
 186 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
 187 {
 188         u32 pio_xfer_ctrl;
 189
 190         pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
 191         pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
 192         pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
 193         writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
 194 }
 195
 196 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
 197 {
 198         u32 ints;
 199
 200         qcom_qspi_pio_xfer_cfg(ctrl);
 201
 202         /* Ack any previous interrupts that might be hanging around */
 203         writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
 204
 205         /* Setup new interrupts */
 206         if (ctrl->xfer.dir == QSPI_WRITE)
 207                 ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
 208         else
 209                 ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
 210         writel(ints, ctrl->base + MSTR_INT_EN);
 211
 212         /* Kick off the transfer */
 213         qcom_qspi_pio_xfer_ctrl(ctrl);
 214 }
 215
 216 static void qcom_qspi_handle_err(struct spi_master *master,
 217                                  struct spi_message *msg)
 218 {
 219         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 220         unsigned long flags;
 221
 222         spin_lock_irqsave(&ctrl->lock, flags);
 223         writel(0, ctrl->base + MSTR_INT_EN);
 224         ctrl->xfer.rem_bytes = 0;
 225         spin_unlock_irqrestore(&ctrl->lock, flags);
 226 }
 227
 228 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 229 {
 230         int ret;
 231         unsigned int avg_bw_cpu;
 232
 233         if (speed_hz == ctrl->last_speed)
 234                 return 0;
 235
 236         /* In regular operation (SBL_EN=1) core must be 4x transfer clock */
 237         ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
 238         if (ret) {
 239                 dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
 240                 return ret;
 241         }
 242
 243         /*
 244          * Set BW quota for CPU as driver supports FIFO mode only.
 245          * We don't have explicit peak requirement so keep it equal to avg_bw.
 246          */
 247         avg_bw_cpu = Bps_to_icc(speed_hz);
 248         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
 249         if (ret) {
 250                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 251                         __func__, ret);
 252                 return ret;
 253         }
 254
 255         ctrl->last_speed = speed_hz;
 256
 257         return 0;
 258 }
 259
 260 static int qcom_qspi_transfer_one(struct spi_master *master,
 261                                   struct spi_device *slv,
 262                                   struct spi_transfer *xfer)
 263 {
 264         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 265         int ret;
 266         unsigned long speed_hz;
 267         unsigned long flags;
 268
 269         speed_hz = slv->max_speed_hz;
 270         if (xfer->speed_hz)
 271                 speed_hz = xfer->speed_hz;
 272
 273         ret = qcom_qspi_set_speed(ctrl, speed_hz);
 274         if (ret)
 275                 return ret;
 276
 277         spin_lock_irqsave(&ctrl->lock, flags);
 278
 279         /* We are half duplex, so either rx or tx will be set */
 280         if (xfer->rx_buf) {
 281                 ctrl->xfer.dir = QSPI_READ;
 282                 ctrl->xfer.buswidth = xfer->rx_nbits;
 283                 ctrl->xfer.rx_buf = xfer->rx_buf;
 284         } else {
 285                 ctrl->xfer.dir = QSPI_WRITE;
 286                 ctrl->xfer.buswidth = xfer->tx_nbits;
 287                 ctrl->xfer.tx_buf = xfer->tx_buf;
 288         }
 289         ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
 290                                           &master->cur_msg->transfers);
 291         ctrl->xfer.rem_bytes = xfer->len;
 292         qcom_qspi_pio_xfer(ctrl);
 293
 294         spin_unlock_irqrestore(&ctrl->lock, flags);
 295
 296         /* We'll call spi_finalize_current_transfer() when done */
 297         return 1;
 298 }
 299
 300 static int qcom_qspi_prepare_message(struct spi_master *master,
 301                                      struct spi_message *message)
 302 {
 303         u32 mstr_cfg;
 304         struct qcom_qspi *ctrl;
 305         int tx_data_oe_delay = 1;
 306         int tx_data_delay = 1;
 307         unsigned long flags;
 308
 309         ctrl = spi_master_get_devdata(master);
 310         spin_lock_irqsave(&ctrl->lock, flags);
 311
 312         mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
 313         mstr_cfg &= ~CHIP_SELECT_NUM;
 314         if (message->spi->chip_select)
 315                 mstr_cfg |= CHIP_SELECT_NUM;
 316
 317         mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
 318         mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
 319         mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
 320         mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
 321         mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
 322         mstr_cfg &= ~DMA_ENABLE;
 323
 324         writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
 325         spin_unlock_irqrestore(&ctrl->lock, flags);
 326
 327         return 0;
 328 }
 329
 330 static irqreturn_t pio_read(struct qcom_qspi *ctrl)
 331 {
 332         u32 rd_fifo_status;
 333         u32 rd_fifo;
 334         unsigned int wr_cnts;
 335         unsigned int bytes_to_read;
 336         unsigned int words_to_read;
 337         u32 *word_buf;
 338         u8 *byte_buf;
 339         int i;
 340
 341         rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
 342
 343         if (!(rd_fifo_status & FIFO_RDY)) {
 344                 dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
 345                 return IRQ_NONE;
 346         }
 347
 348         wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
 349         wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
 350
 351         words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
 352         bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
 353
 354         if (words_to_read) {
 355                 word_buf = ctrl->xfer.rx_buf;
 356                 ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
 357                 ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
 358                 ctrl->xfer.rx_buf = word_buf + words_to_read;
 359         }
 360
 361         if (bytes_to_read) {
 362                 byte_buf = ctrl->xfer.rx_buf;
 363                 rd_fifo = readl(ctrl->base + RD_FIFO);
 364                 ctrl->xfer.rem_bytes -= bytes_to_read;
 365                 for (i = 0; i < bytes_to_read; i++)
 366                         *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
 367                 ctrl->xfer.rx_buf = byte_buf;
 368         }
 369
 370         return IRQ_HANDLED;
 371 }
 372
 373 static irqreturn_t pio_write(struct qcom_qspi *ctrl)
 374 {
 375         const void *xfer_buf = ctrl->xfer.tx_buf;
 376         const int *word_buf;
 377         const char *byte_buf;
 378         unsigned int wr_fifo_bytes;
 379         unsigned int wr_fifo_words;
 380         unsigned int wr_size;
 381         unsigned int rem_words;
 382
 383         wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
 384         wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
 385
 386         if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
 387                 /* Process the last 1-3 bytes */
 388                 wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
 389                 ctrl->xfer.rem_bytes -= wr_size;
 390
 391                 byte_buf = xfer_buf;
 392                 while (wr_size--)
 393                         writel(*byte_buf++,
 394                                ctrl->base + PIO_DATAOUT_1B);
 395                 ctrl->xfer.tx_buf = byte_buf;
 396         } else {
 397                 /*
 398                  * Process all the whole words; to keep things simple we'll
 399                  * just wait for the next interrupt to handle the last 1-3
 400                  * bytes if we don't have an even number of words.
 401                  */
 402                 rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
 403                 wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
 404
 405                 wr_size = min(rem_words, wr_fifo_words);
 406                 ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
 407
 408                 word_buf = xfer_buf;
 409                 iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
 410                 ctrl->xfer.tx_buf = word_buf + wr_size;
 411
 412         }
 413
 414         return IRQ_HANDLED;
 415 }
 416
 417 static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
 418 {
 419         u32 int_status;
 420         struct qcom_qspi *ctrl = dev_id;
 421         irqreturn_t ret = IRQ_NONE;
 422
 423         spin_lock(&ctrl->lock);
 424
 425         int_status = readl(ctrl->base + MSTR_INT_STATUS);
 426         writel(int_status, ctrl->base + MSTR_INT_STATUS);
 427
 428         if (ctrl->xfer.dir == QSPI_WRITE) {
 429                 if (int_status & WR_FIFO_EMPTY)
 430                         ret = pio_write(ctrl);
 431         } else {
 432                 if (int_status & RESP_FIFO_RDY)
 433                         ret = pio_read(ctrl);
 434         }
 435
 436         if (int_status & QSPI_ERR_IRQS) {
 437                 if (int_status & RESP_FIFO_UNDERRUN)
 438                         dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
 439                 if (int_status & WR_FIFO_OVERRUN)
 440                         dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
 441                 if (int_status & HRESP_FROM_NOC_ERR)
 442                         dev_err(ctrl->dev, "IRQ error: NOC response error\n");
 443                 ret = IRQ_HANDLED;
 444         }
 445
 446         if (!ctrl->xfer.rem_bytes) {
 447                 writel(0, ctrl->base + MSTR_INT_EN);
 448                 spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
 449         }
 450
 451         spin_unlock(&ctrl->lock);
 452         return ret;
 453 }
 454
 455 static int qcom_qspi_probe(struct platform_device *pdev)
 456 {
 457         int ret;
 458         struct device *dev;
 459         struct spi_master *master;
 460         struct qcom_qspi *ctrl;
 461
 462         dev = &pdev->dev;
 463
 464         master = devm_spi_alloc_master(dev, sizeof(*ctrl));
 465         if (!master)
 466                 return -ENOMEM;
 467
 468         platform_set_drvdata(pdev, master);
 469
 470         ctrl = spi_master_get_devdata(master);
 471
 472         spin_lock_init(&ctrl->lock);
 473         ctrl->dev = dev;
 474         ctrl->base = devm_platform_ioremap_resource(pdev, 0);
 475         if (IS_ERR(ctrl->base))
 476                 return PTR_ERR(ctrl->base);
 477
 478         ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
 479                                   sizeof(*ctrl->clks), GFP_KERNEL);
 480         if (!ctrl->clks)
 481                 return -ENOMEM;
 482
 483         ctrl->clks[QSPI_CLK_CORE].id = "core";
 484         ctrl->clks[QSPI_CLK_IFACE].id = "iface";
 485         ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
 486         if (ret)
 487                 return ret;
 488
 489         ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
 490         if (IS_ERR(ctrl->icc_path_cpu_to_qspi))
 491                 return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
 492                                      "Failed to get cpu path\n");
 493
 494         /* Set BW vote for register access */
 495         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
 496                                 Bps_to_icc(1000));
 497         if (ret) {
 498                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 499                                 __func__, ret);
 500                 return ret;
 501         }
 502
 503         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 504         if (ret) {
 505                 dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 506                                 __func__, ret);
 507                 return ret;
 508         }
 509
 510         ret = platform_get_irq(pdev, 0);
 511         if (ret < 0)
 512                 return ret;
 513         ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);
 514         if (ret) {
 515                 dev_err(dev, "Failed to request irq %d\n", ret);
 516                 return ret;
 517         }
 518
 519         master->max_speed_hz = 300000000;
 520         master->num_chipselect = QSPI_NUM_CS;
 521         master->bus_num = -1;
 522         master->dev.of_node = pdev->dev.of_node;
 523         master->mode_bits = SPI_MODE_0 |
 524                             SPI_TX_DUAL | SPI_RX_DUAL |
 525                             SPI_TX_QUAD | SPI_RX_QUAD;
 526         master->flags = SPI_MASTER_HALF_DUPLEX;
 527         master->prepare_message = qcom_qspi_prepare_message;
 528         master->transfer_one = qcom_qspi_transfer_one;
 529         master->handle_err = qcom_qspi_handle_err;
 530         master->auto_runtime_pm = true;
 531
 532         ret = devm_pm_opp_set_clkname(&pdev->dev, "core");
 533         if (ret)
 534                 return ret;
 535         /* OPP table is optional */
 536         ret = devm_pm_opp_of_add_table(&pdev->dev);
 537         if (ret && ret != -ENODEV) {
 538                 dev_err(&pdev->dev, "invalid OPP table in device tree\n");
 539                 return ret;
 540         }
 541
 542         pm_runtime_use_autosuspend(dev);
 543         pm_runtime_set_autosuspend_delay(dev, 250);
 544         pm_runtime_enable(dev);
 545
 546         ret = spi_register_master(master);
 547         if (!ret)
 548                 return 0;
 549
 550         pm_runtime_disable(dev);
 551
 552         return ret;
 553 }
 554
 555 static int qcom_qspi_remove(struct platform_device *pdev)
 556 {
 557         struct spi_master *master = platform_get_drvdata(pdev);
 558
 559         /* Unregister _before_ disabling pm_runtime() so we stop transfers */
 560         spi_unregister_master(master);
 561
 562         pm_runtime_disable(&pdev->dev);
 563
 564         return 0;
 565 }
 566
 567 static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
 568 {
 569         struct spi_master *master = dev_get_drvdata(dev);
 570         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 571         int ret;
 572
 573         /* Drop the performance state vote */
 574         dev_pm_opp_set_rate(dev, 0);
 575         clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
 576
 577         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 578         if (ret) {
 579                 dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 580                         __func__, ret);
 581                 return ret;
 582         }
 583
 584         return 0;
 585 }
 586
 587 static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
 588 {
 589         struct spi_master *master = dev_get_drvdata(dev);
 590         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 591         int ret;
 592
 593         ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
 594         if (ret) {
 595                 dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
 596                         __func__, ret);
 597                 return ret;
 598         }
 599
 600         ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
 601         if (ret)
 602                 return ret;
 603
 604         return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
 605 }
 606
 607 static int __maybe_unused qcom_qspi_suspend(struct device *dev)
 608 {
 609         struct spi_master *master = dev_get_drvdata(dev);
 610         int ret;
 611
 612         ret = spi_master_suspend(master);
 613         if (ret)
 614                 return ret;
 615
 616         ret = pm_runtime_force_suspend(dev);
 617         if (ret)
 618                 spi_master_resume(master);
 619
 620         return ret;
 621 }
 622
 623 static int __maybe_unused qcom_qspi_resume(struct device *dev)
 624 {
 625         struct spi_master *master = dev_get_drvdata(dev);
 626         int ret;
 627
 628         ret = pm_runtime_force_resume(dev);
 629         if (ret)
 630                 return ret;
 631
 632         ret = spi_master_resume(master);
 633         if (ret)
 634                 pm_runtime_force_suspend(dev);
 635
 636         return ret;
 637 }
 638
 639 static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
 640         SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
 641                            qcom_qspi_runtime_resume, NULL)
 642         SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
 643 };
 644
 645 static const struct of_device_id qcom_qspi_dt_match[] = {
 646         { .compatible = "qcom,qspi-v1", },
 647         { }
 648 };
 649 MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
 650
 651 static struct platform_driver qcom_qspi_driver = {
 652         .driver = {
 653                 .name           = "qcom_qspi",
 654                 .pm             = &qcom_qspi_dev_pm_ops,
 655                 .of_match_table = qcom_qspi_dt_match,
 656         },
 657         .probe = qcom_qspi_probe,
 658         .remove = qcom_qspi_remove,
 659 };
 660 module_platform_driver(qcom_qspi_driver);
 661
 662 MODULE_DESCRIPTION("SPI driver for QSPI cores");
 663 MODULE_LICENSE("GPL v2");