drivers/mmc/host/bcm2835-sdhost.c

   1 /*
   2  * BCM2835 SD host driver.
   3  *
   4  * Author:      Phil Elwell <phil@raspberrypi.org>
   5  *              Copyright (C) 2015-2016 Raspberry Pi (Trading) Ltd.
   6  *
   7  * Based on
   8  *  mmc-bcm2835.c by Gellert Weisz
   9  * which is, in turn, based on
  10  *  sdhci-bcm2708.c by Broadcom
  11  *  sdhci-bcm2835.c by Stephen Warren and Oleksandr Tymoshenko
  12  *  sdhci.c and sdhci-pci.c by Pierre Ossman
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms and conditions of the GNU General Public License,
  16  * version 2, as published by the Free Software Foundation.
  17  *
  18  * This program is distributed in the hope it will be useful, but WITHOUT
  19  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  20  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  21  * more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  25  */
  26
  27 #define FIFO_READ_THRESHOLD     4
  28 #define FIFO_WRITE_THRESHOLD    4
  29 #define ALLOW_CMD23_READ        1
  30 #define ALLOW_CMD23_WRITE       0
  31 #define ENABLE_LOG              1
  32 #define SDDATA_FIFO_PIO_BURST   8
  33 #define CMD_DALLY_US            1
  34
  35 #include <linux/delay.h>
  36 #include <linux/module.h>
  37 #include <linux/io.h>
  38 #include <linux/mmc/mmc.h>
  39 #include <linux/mmc/host.h>
  40 #include <linux/mmc/sd.h>
  41 #include <linux/mmc/sdio.h>
  42 #include <linux/scatterlist.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_irq.h>
  45 #include <linux/clk.h>
  46 #include <linux/platform_device.h>
  47 #include <linux/err.h>
  48 #include <linux/blkdev.h>
  49 #include <linux/dmaengine.h>
  50 #include <linux/dma-mapping.h>
  51 #include <linux/of_dma.h>
  52 #include <linux/time.h>
  53 #include <linux/workqueue.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/highmem.h>
  56 #include <soc/bcm2835/raspberrypi-firmware.h>
  57
  58 /* For mmc_card_blockaddr */
  59 #include "../core/card.h"
  60
  61 #define DRIVER_NAME "sdhost-bcm2835"
  62
  63 #define SDCMD  0x00 /* Command to SD card              - 16 R/W */
  64 #define SDARG  0x04 /* Argument to SD card             - 32 R/W */
  65 #define SDTOUT 0x08 /* Start value for timeout counter - 32 R/W */
  66 #define SDCDIV 0x0c /* Start value for clock divider   - 11 R/W */
  67 #define SDRSP0 0x10 /* SD card response (31:0)         - 32 R   */
  68 #define SDRSP1 0x14 /* SD card response (63:32)        - 32 R   */
  69 #define SDRSP2 0x18 /* SD card response (95:64)        - 32 R   */
  70 #define SDRSP3 0x1c /* SD card response (127:96)       - 32 R   */
  71 #define SDHSTS 0x20 /* SD host status                  - 11 R   */
  72 #define SDVDD  0x30 /* SD card power control           -  1 R/W */
  73 #define SDEDM  0x34 /* Emergency Debug Mode            - 13 R/W */
  74 #define SDHCFG 0x38 /* Host configuration              -  2 R/W */
  75 #define SDHBCT 0x3c /* Host byte count (debug)         - 32 R/W */
  76 #define SDDATA 0x40 /* Data to/from SD card            - 32 R/W */
  77 #define SDHBLC 0x50 /* Host block count (SDIO/SDHC)    -  9 R/W */
  78
  79 #define SDCMD_NEW_FLAG                  0x8000
  80 #define SDCMD_FAIL_FLAG                 0x4000
  81 #define SDCMD_BUSYWAIT                  0x800
  82 #define SDCMD_NO_RESPONSE               0x400
  83 #define SDCMD_LONG_RESPONSE             0x200
  84 #define SDCMD_WRITE_CMD                 0x80
  85 #define SDCMD_READ_CMD                  0x40
  86 #define SDCMD_CMD_MASK                  0x3f
  87
  88 #define SDCDIV_MAX_CDIV                 0x7ff
  89
  90 #define SDHSTS_BUSY_IRPT                0x400
  91 #define SDHSTS_BLOCK_IRPT               0x200
  92 #define SDHSTS_SDIO_IRPT                0x100
  93 #define SDHSTS_REW_TIME_OUT             0x80
  94 #define SDHSTS_CMD_TIME_OUT             0x40
  95 #define SDHSTS_CRC16_ERROR              0x20
  96 #define SDHSTS_CRC7_ERROR               0x10
  97 #define SDHSTS_FIFO_ERROR               0x08
  98 /* Reserved */
  99 /* Reserved */
 100 #define SDHSTS_DATA_FLAG                0x01
 101
 102 #define SDHSTS_TRANSFER_ERROR_MASK      (SDHSTS_CRC7_ERROR|SDHSTS_CRC16_ERROR|SDHSTS_REW_TIME_OUT|SDHSTS_FIFO_ERROR)
 103 #define SDHSTS_ERROR_MASK               (SDHSTS_CMD_TIME_OUT|SDHSTS_TRANSFER_ERROR_MASK)
 104
 105 #define SDHCFG_BUSY_IRPT_EN     (1<<10)
 106 #define SDHCFG_BLOCK_IRPT_EN    (1<<8)
 107 #define SDHCFG_SDIO_IRPT_EN     (1<<5)
 108 #define SDHCFG_DATA_IRPT_EN     (1<<4)
 109 #define SDHCFG_SLOW_CARD        (1<<3)
 110 #define SDHCFG_WIDE_EXT_BUS     (1<<2)
 111 #define SDHCFG_WIDE_INT_BUS     (1<<1)
 112 #define SDHCFG_REL_CMD_LINE     (1<<0)
 113
 114 #define SDEDM_FORCE_DATA_MODE   (1<<19)
 115 #define SDEDM_CLOCK_PULSE       (1<<20)
 116 #define SDEDM_BYPASS            (1<<21)
 117
 118 #define SDEDM_WRITE_THRESHOLD_SHIFT 9
 119 #define SDEDM_READ_THRESHOLD_SHIFT 14
 120 #define SDEDM_THRESHOLD_MASK     0x1f
 121
 122 #define SDEDM_FSM_MASK           0xf
 123 #define SDEDM_FSM_IDENTMODE      0x0
 124 #define SDEDM_FSM_DATAMODE       0x1
 125 #define SDEDM_FSM_READDATA       0x2
 126 #define SDEDM_FSM_WRITEDATA      0x3
 127 #define SDEDM_FSM_READWAIT       0x4
 128 #define SDEDM_FSM_READCRC        0x5
 129 #define SDEDM_FSM_WRITECRC       0x6
 130 #define SDEDM_FSM_WRITEWAIT1     0x7
 131 #define SDEDM_FSM_POWERDOWN      0x8
 132 #define SDEDM_FSM_POWERUP        0x9
 133 #define SDEDM_FSM_WRITESTART1    0xa
 134 #define SDEDM_FSM_WRITESTART2    0xb
 135 #define SDEDM_FSM_GENPULSES      0xc
 136 #define SDEDM_FSM_WRITEWAIT2     0xd
 137 #define SDEDM_FSM_STARTPOWDOWN   0xf
 138
 139 #define SDDATA_FIFO_WORDS        16
 140
 141 #define USE_CMD23_FLAGS          ((ALLOW_CMD23_READ * MMC_DATA_READ) | \
 142                                   (ALLOW_CMD23_WRITE * MMC_DATA_WRITE))
 143
 144 #define MHZ 1000000
 145
 146
 147 struct bcm2835_host {
 148         spinlock_t              lock;
 149
 150         struct rpi_firmware     *fw;
 151
 152         void __iomem            *ioaddr;
 153         phys_addr_t             bus_addr;
 154
 155         struct mmc_host         *mmc;
 156
 157         u32                     pio_timeout;    /* In jiffies */
 158
 159         int                     clock;          /* Current clock speed */
 160
 161         bool                    slow_card;      /* Force 11-bit divisor */
 162
 163         unsigned int            max_clk;        /* Max possible freq */
 164
 165         struct tasklet_struct   finish_tasklet; /* Tasklet structures */
 166
 167         struct work_struct      cmd_wait_wq;    /* Workqueue function */
 168
 169         struct timer_list       timer;          /* Timer for timeouts */
 170
 171         struct sg_mapping_iter  sg_miter;       /* SG state for PIO */
 172         unsigned int            blocks;         /* remaining PIO blocks */
 173
 174         int                     irq;            /* Device IRQ */
 175
 176         u32                     cmd_quick_poll_retries;
 177         u32                     ns_per_fifo_word;
 178
 179         /* cached registers */
 180         u32                     hcfg;
 181         u32                     cdiv;
 182
 183         struct mmc_request              *mrq;                   /* Current request */
 184         struct mmc_command              *cmd;                   /* Current command */
 185         struct mmc_data                 *data;                  /* Current data request */
 186         unsigned int                    data_complete:1;        /* Data finished before cmd */
 187
 188         unsigned int                    flush_fifo:1;           /* Drain the fifo when finishing */
 189
 190         unsigned int                    use_busy:1;             /* Wait for busy interrupt */
 191
 192         unsigned int                    use_sbc:1;              /* Send CMD23 */
 193
 194         unsigned int                    debug:1;                /* Enable debug output */
 195         unsigned int                    firmware_sets_cdiv:1;   /* Let the firmware manage the clock */
 196         unsigned int                    reset_clock:1;          /* Reset the clock fore the next request */
 197
 198         /*DMA part*/
 199         struct dma_chan                 *dma_chan_rxtx;         /* DMA channel for reads and writes */
 200         struct dma_chan                 *dma_chan;              /* Channel in use */
 201         struct dma_slave_config         dma_cfg_rx;
 202         struct dma_slave_config         dma_cfg_tx;
 203         struct dma_async_tx_descriptor  *dma_desc;
 204         u32                             dma_dir;
 205         u32                             drain_words;
 206         struct page                     *drain_page;
 207         u32                             drain_offset;
 208
 209         bool                            allow_dma;
 210         bool                            use_dma;
 211         /*end of DMA part*/
 212
 213         int                             max_delay;      /* maximum length of time spent waiting */
 214         struct timespec64               stop_time;      /* when the last stop was issued */
 215         u32                             delay_after_stop; /* minimum time between stop and subsequent data transfer */
 216         u32                             delay_after_this_stop; /* minimum time between this stop and subsequent data transfer */
 217         u32                             user_overclock_50; /* User's preferred frequency to use when 50MHz is requested (in MHz) */
 218         u32                             overclock_50;   /* frequency to use when 50MHz is requested (in MHz) */
 219         u32                             overclock;      /* Current frequency if overclocked, else zero */
 220         u32                             pio_limit;      /* Maximum block count for PIO (0 = always DMA) */
 221
 222         u32                             sectors;        /* Cached card size in sectors */
 223 };
 224
 225 #if ENABLE_LOG
 226
 227 struct log_entry_struct {
 228         char event[4];
 229         u32 timestamp;
 230         u32 param1;
 231         u32 param2;
 232 };
 233
 234 typedef struct log_entry_struct LOG_ENTRY_T;
 235
 236 LOG_ENTRY_T *sdhost_log_buf;
 237 dma_addr_t sdhost_log_addr;
 238 static u32 sdhost_log_idx;
 239 static spinlock_t log_lock;
 240 static void __iomem *timer_base;
 241
 242 #define LOG_ENTRIES (256*1)
 243 #define LOG_SIZE (sizeof(LOG_ENTRY_T)*LOG_ENTRIES)
 244
 245 static void log_init(struct device *dev, u32 bus_to_phys)
 246 {
 247         spin_lock_init(&log_lock);
 248         sdhost_log_buf = dma_alloc_coherent(dev, LOG_SIZE, &sdhost_log_addr,
 249                                              GFP_KERNEL);
 250         if (sdhost_log_buf) {
 251                 pr_info("sdhost: log_buf @ %p (%llx)\n",
 252                         sdhost_log_buf, (u64)sdhost_log_addr);
 253                 timer_base = ioremap(bus_to_phys + 0x7e003000, SZ_4K);
 254                 if (!timer_base)
 255                         pr_err("sdhost: failed to remap timer\n");
 256         }
 257         else
 258                 pr_err("sdhost: failed to allocate log buf\n");
 259 }
 260
 261 static void log_event_impl(const char *event, u32 param1, u32 param2)
 262 {
 263         if (sdhost_log_buf) {
 264                 LOG_ENTRY_T *entry;
 265                 unsigned long flags;
 266
 267                 spin_lock_irqsave(&log_lock, flags);
 268
 269                 entry = sdhost_log_buf + sdhost_log_idx;
 270                 memcpy(entry->event, event, 4);
 271                 entry->timestamp = (readl(timer_base + 4) & 0x3fffffff) +
 272                         (smp_processor_id()<<30);
 273                 entry->param1 = param1;
 274                 entry->param2 = param2;
 275                 sdhost_log_idx = (sdhost_log_idx + 1) % LOG_ENTRIES;
 276
 277                 spin_unlock_irqrestore(&log_lock, flags);
 278         }
 279 }
 280
 281 static void log_dump(void)
 282 {
 283         if (sdhost_log_buf) {
 284                 LOG_ENTRY_T *entry;
 285                 unsigned long flags;
 286                 int idx;
 287
 288                 spin_lock_irqsave(&log_lock, flags);
 289
 290                 idx = sdhost_log_idx;
 291                 do {
 292                         entry = sdhost_log_buf + idx;
 293                         if (entry->event[0] != '\0')
 294                                 pr_info("[%08x] %.4s %x %x\n",
 295                                        entry->timestamp,
 296                                        entry->event,
 297                                        entry->param1,
 298                                        entry->param2);
 299                         idx = (idx + 1) % LOG_ENTRIES;
 300                 } while (idx != sdhost_log_idx);
 301
 302                 spin_unlock_irqrestore(&log_lock, flags);
 303         }
 304 }
 305
 306 #define log_event(event, param1, param2) log_event_impl(event, (u32)(uintptr_t)param1, (u32)(uintptr_t)param2)
 307
 308 #else
 309
 310 #define log_init(x) (void)0
 311 #define log_event(event, param1, param2) (void)0
 312 #define log_dump() (void)0
 313
 314 #endif
 315
 316 static inline void bcm2835_sdhost_write(struct bcm2835_host *host, u32 val, int reg)
 317 {
 318         writel(val, host->ioaddr + reg);
 319 }
 320
 321 static inline u32 bcm2835_sdhost_read(struct bcm2835_host *host, int reg)
 322 {
 323         return readl(host->ioaddr + reg);
 324 }
 325
 326 static inline u32 bcm2835_sdhost_read_relaxed(struct bcm2835_host *host, int reg)
 327 {
 328         return readl_relaxed(host->ioaddr + reg);
 329 }
 330
 331 static void bcm2835_sdhost_dumpcmd(struct bcm2835_host *host,
 332                                    struct mmc_command *cmd,
 333                                    const char *label)
 334 {
 335         if (cmd)
 336                 pr_info("%s:%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
 337                         mmc_hostname(host->mmc),
 338                         (cmd == host->cmd) ? '>' : ' ',
 339                         label, cmd->opcode, cmd->arg, cmd->flags,
 340                         cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3],
 341                         cmd->error);
 342 }
 343
 344 static void bcm2835_sdhost_dumpregs(struct bcm2835_host *host)
 345 {
 346         if (host->mrq)
 347         {
 348                 bcm2835_sdhost_dumpcmd(host, host->mrq->sbc, "sbc");
 349                 bcm2835_sdhost_dumpcmd(host, host->mrq->cmd, "cmd");
 350                 if (host->mrq->data)
 351                         pr_info("%s: data blocks %x blksz %x - err %d\n",
 352                                mmc_hostname(host->mmc),
 353                                host->mrq->data->blocks,
 354                                host->mrq->data->blksz,
 355                                host->mrq->data->error);
 356                 bcm2835_sdhost_dumpcmd(host, host->mrq->stop, "stop");
 357         }
 358
 359         pr_info("%s: =========== REGISTER DUMP ===========\n",
 360                 mmc_hostname(host->mmc));
 361
 362         pr_info("%s: SDCMD  0x%08x\n",
 363                 mmc_hostname(host->mmc),
 364                 bcm2835_sdhost_read(host, SDCMD));
 365         pr_info("%s: SDARG  0x%08x\n",
 366                 mmc_hostname(host->mmc),
 367                 bcm2835_sdhost_read(host, SDARG));
 368         pr_info("%s: SDTOUT 0x%08x\n",
 369                 mmc_hostname(host->mmc),
 370                 bcm2835_sdhost_read(host, SDTOUT));
 371         pr_info("%s: SDCDIV 0x%08x\n",
 372                 mmc_hostname(host->mmc),
 373                 bcm2835_sdhost_read(host, SDCDIV));
 374         pr_info("%s: SDRSP0 0x%08x\n",
 375                 mmc_hostname(host->mmc),
 376                 bcm2835_sdhost_read(host, SDRSP0));
 377         pr_info("%s: SDRSP1 0x%08x\n",
 378                 mmc_hostname(host->mmc),
 379                 bcm2835_sdhost_read(host, SDRSP1));
 380         pr_info("%s: SDRSP2 0x%08x\n",
 381                 mmc_hostname(host->mmc),
 382                 bcm2835_sdhost_read(host, SDRSP2));
 383         pr_info("%s: SDRSP3 0x%08x\n",
 384                 mmc_hostname(host->mmc),
 385                 bcm2835_sdhost_read(host, SDRSP3));
 386         pr_info("%s: SDHSTS 0x%08x\n",
 387                 mmc_hostname(host->mmc),
 388                 bcm2835_sdhost_read(host, SDHSTS));
 389         pr_info("%s: SDVDD  0x%08x\n",
 390                 mmc_hostname(host->mmc),
 391                 bcm2835_sdhost_read(host, SDVDD));
 392         pr_info("%s: SDEDM  0x%08x\n",
 393                 mmc_hostname(host->mmc),
 394                 bcm2835_sdhost_read(host, SDEDM));
 395         pr_info("%s: SDHCFG 0x%08x\n",
 396                 mmc_hostname(host->mmc),
 397                 bcm2835_sdhost_read(host, SDHCFG));
 398         pr_info("%s: SDHBCT 0x%08x\n",
 399                 mmc_hostname(host->mmc),
 400                 bcm2835_sdhost_read(host, SDHBCT));
 401         pr_info("%s: SDHBLC 0x%08x\n",
 402                 mmc_hostname(host->mmc),
 403                 bcm2835_sdhost_read(host, SDHBLC));
 404
 405         pr_info("%s: ===========================================\n",
 406                 mmc_hostname(host->mmc));
 407 }
 408
 409 static void bcm2835_sdhost_set_power(struct bcm2835_host *host, bool on)
 410 {
 411         bcm2835_sdhost_write(host, on ? 1 : 0, SDVDD);
 412 }
 413
 414 static void bcm2835_sdhost_reset_internal(struct bcm2835_host *host)
 415 {
 416         u32 temp;
 417
 418         if (host->debug)
 419                 pr_info("%s: reset\n", mmc_hostname(host->mmc));
 420
 421         bcm2835_sdhost_set_power(host, false);
 422
 423         bcm2835_sdhost_write(host, 0, SDCMD);
 424         bcm2835_sdhost_write(host, 0, SDARG);
 425         bcm2835_sdhost_write(host, 0xf00000, SDTOUT);
 426         bcm2835_sdhost_write(host, 0, SDCDIV);
 427         bcm2835_sdhost_write(host, 0x7f8, SDHSTS); /* Write 1s to clear */
 428         bcm2835_sdhost_write(host, 0, SDHCFG);
 429         bcm2835_sdhost_write(host, 0, SDHBCT);
 430         bcm2835_sdhost_write(host, 0, SDHBLC);
 431
 432         /* Limit fifo usage due to silicon bug */
 433         temp = bcm2835_sdhost_read(host, SDEDM);
 434         temp &= ~((SDEDM_THRESHOLD_MASK<<SDEDM_READ_THRESHOLD_SHIFT) |
 435                   (SDEDM_THRESHOLD_MASK<<SDEDM_WRITE_THRESHOLD_SHIFT));
 436         temp |= (FIFO_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
 437                 (FIFO_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
 438         bcm2835_sdhost_write(host, temp, SDEDM);
 439         mdelay(10);
 440         bcm2835_sdhost_set_power(host, true);
 441         mdelay(10);
 442         host->clock = 0;
 443         host->sectors = 0;
 444         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 445         bcm2835_sdhost_write(host, SDCDIV_MAX_CDIV, SDCDIV);
 446 }
 447
 448 static void bcm2835_sdhost_reset(struct mmc_host *mmc)
 449 {
 450         struct bcm2835_host *host = mmc_priv(mmc);
 451         unsigned long flags;
 452         spin_lock_irqsave(&host->lock, flags);
 453         log_event("RST<", 0, 0);
 454
 455         bcm2835_sdhost_reset_internal(host);
 456
 457         spin_unlock_irqrestore(&host->lock, flags);
 458 }
 459
 460 static void bcm2835_sdhost_set_ios(struct mmc_host *mmc, struct mmc_ios *ios);
 461
 462 static void bcm2835_sdhost_init(struct bcm2835_host *host, int soft)
 463 {
 464         pr_debug("bcm2835_sdhost_init(%d)\n", soft);
 465
 466         /* Set interrupt enables */
 467         host->hcfg = SDHCFG_BUSY_IRPT_EN;
 468
 469         bcm2835_sdhost_reset_internal(host);
 470
 471         if (soft) {
 472                 /* force clock reconfiguration */
 473                 host->clock = 0;
 474                 bcm2835_sdhost_set_ios(host->mmc, &host->mmc->ios);
 475         }
 476 }
 477
 478 static void bcm2835_sdhost_wait_transfer_complete(struct bcm2835_host *host)
 479 {
 480         int timediff;
 481         u32 alternate_idle;
 482         u32 edm;
 483
 484         alternate_idle = (host->mrq->data->flags & MMC_DATA_READ) ?
 485                 SDEDM_FSM_READWAIT : SDEDM_FSM_WRITESTART1;
 486
 487         edm = bcm2835_sdhost_read(host, SDEDM);
 488
 489         log_event("WTC<", edm, 0);
 490
 491         timediff = 0;
 492
 493         while (1) {
 494                 u32 fsm = edm & SDEDM_FSM_MASK;
 495                 if ((fsm == SDEDM_FSM_IDENTMODE) ||
 496                     (fsm == SDEDM_FSM_DATAMODE))
 497                         break;
 498                 if (fsm == alternate_idle) {
 499                         bcm2835_sdhost_write(host,
 500                                              edm | SDEDM_FORCE_DATA_MODE,
 501                                              SDEDM);
 502                         break;
 503                 }
 504
 505                 timediff++;
 506                 if (timediff == 100000) {
 507                         pr_err("%s: wait_transfer_complete - still waiting after %d retries\n",
 508                                mmc_hostname(host->mmc),
 509                                timediff);
 510                         log_dump();
 511                         bcm2835_sdhost_dumpregs(host);
 512                         host->mrq->data->error = -ETIMEDOUT;
 513                         log_event("WTC!", edm, 0);
 514                         return;
 515                 }
 516                 cpu_relax();
 517                 edm = bcm2835_sdhost_read(host, SDEDM);
 518         }
 519         log_event("WTC>", edm, 0);
 520 }
 521
 522 static void bcm2835_sdhost_finish_data(struct bcm2835_host *host);
 523
 524 static void bcm2835_sdhost_dma_complete(void *param)
 525 {
 526         struct bcm2835_host *host = param;
 527         struct mmc_data *data = host->data;
 528         unsigned long flags;
 529
 530         spin_lock_irqsave(&host->lock, flags);
 531         log_event("DMA<", host->data, bcm2835_sdhost_read(host, SDHSTS));
 532         log_event("DMA ", bcm2835_sdhost_read(host, SDCMD),
 533                   bcm2835_sdhost_read(host, SDEDM));
 534
 535         if (host->dma_chan) {
 536                 dma_unmap_sg(host->dma_chan->device->dev,
 537                              data->sg, data->sg_len,
 538                              host->dma_dir);
 539
 540                 host->dma_chan = NULL;
 541         }
 542
 543         if (host->drain_words) {
 544                 void *page;
 545                 u32 *buf;
 546
 547                 if (host->drain_offset & PAGE_MASK) {
 548                         host->drain_page += host->drain_offset >> PAGE_SHIFT;
 549                         host->drain_offset &= ~PAGE_MASK;
 550                 }
 551
 552                 page = kmap_atomic(host->drain_page);
 553                 buf = page + host->drain_offset;
 554
 555                 while (host->drain_words) {
 556                         u32 edm = bcm2835_sdhost_read(host, SDEDM);
 557                         if ((edm >> 4) & 0x1f)
 558                                 *(buf++) = bcm2835_sdhost_read(host,
 559                                                                SDDATA);
 560                         host->drain_words--;
 561                 }
 562
 563                 kunmap_atomic(page);
 564         }
 565
 566         bcm2835_sdhost_finish_data(host);
 567
 568         log_event("DMA>", host->data, 0);
 569         spin_unlock_irqrestore(&host->lock, flags);
 570 }
 571
 572 static void bcm2835_sdhost_read_block_pio(struct bcm2835_host *host)
 573 {
 574         unsigned long flags;
 575         size_t blksize, len;
 576         u32 *buf;
 577         unsigned long wait_max;
 578
 579         blksize = host->data->blksz;
 580
 581         wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
 582
 583         local_irq_save(flags);
 584
 585         while (blksize) {
 586                 int copy_words;
 587                 u32 hsts = 0;
 588
 589                 if (!sg_miter_next(&host->sg_miter)) {
 590                         host->data->error = -EINVAL;
 591                         break;
 592                 }
 593
 594                 len = min(host->sg_miter.length, blksize);
 595                 if (len % 4) {
 596                         host->data->error = -EINVAL;
 597                         break;
 598                 }
 599
 600                 blksize -= len;
 601                 host->sg_miter.consumed = len;
 602
 603                 buf = (u32 *)host->sg_miter.addr;
 604
 605                 copy_words = len/4;
 606
 607                 while (copy_words) {
 608                         int burst_words, words;
 609                         u32 edm;
 610
 611                         burst_words = SDDATA_FIFO_PIO_BURST;
 612                         if (burst_words > copy_words)
 613                                 burst_words = copy_words;
 614                         edm = bcm2835_sdhost_read(host, SDEDM);
 615                         words = ((edm >> 4) & 0x1f);
 616
 617                         if (words < burst_words) {
 618                                 int fsm_state = (edm & SDEDM_FSM_MASK);
 619                                 if ((fsm_state != SDEDM_FSM_READDATA) &&
 620                                     (fsm_state != SDEDM_FSM_READWAIT) &&
 621                                     (fsm_state != SDEDM_FSM_READCRC)) {
 622                                         hsts = bcm2835_sdhost_read(host,
 623                                                                    SDHSTS);
 624                                         pr_info("%s: fsm %x, hsts %x\n",
 625                                                mmc_hostname(host->mmc),
 626                                                fsm_state, hsts);
 627                                         if (hsts & SDHSTS_ERROR_MASK)
 628                                                 break;
 629                                 }
 630
 631                                 if (time_after(jiffies, wait_max)) {
 632                                         pr_err("%s: PIO read timeout - EDM %x\n",
 633                                                mmc_hostname(host->mmc),
 634                                                edm);
 635                                         hsts = SDHSTS_REW_TIME_OUT;
 636                                         break;
 637                                 }
 638                                 ndelay((burst_words - words) *
 639                                        host->ns_per_fifo_word);
 640                                 continue;
 641                         } else if (words > copy_words) {
 642                                 words = copy_words;
 643                         }
 644
 645                         copy_words -= words;
 646
 647                         while (words) {
 648                                 *(buf++) = bcm2835_sdhost_read(host, SDDATA);
 649                                 words--;
 650                         }
 651                 }
 652
 653                 if (hsts & SDHSTS_ERROR_MASK)
 654                         break;
 655         }
 656
 657         sg_miter_stop(&host->sg_miter);
 658
 659         local_irq_restore(flags);
 660 }
 661
 662 static void bcm2835_sdhost_write_block_pio(struct bcm2835_host *host)
 663 {
 664         unsigned long flags;
 665         size_t blksize, len;
 666         u32 *buf;
 667         unsigned long wait_max;
 668
 669         blksize = host->data->blksz;
 670
 671         wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
 672
 673         local_irq_save(flags);
 674
 675         while (blksize) {
 676                 int copy_words;
 677                 u32 hsts = 0;
 678
 679                 if (!sg_miter_next(&host->sg_miter)) {
 680                         host->data->error = -EINVAL;
 681                         break;
 682                 }
 683
 684                 len = min(host->sg_miter.length, blksize);
 685                 if (len % 4) {
 686                         host->data->error = -EINVAL;
 687                         break;
 688                 }
 689
 690                 blksize -= len;
 691                 host->sg_miter.consumed = len;
 692
 693                 buf = (u32 *)host->sg_miter.addr;
 694
 695                 copy_words = len/4;
 696
 697                 while (copy_words) {
 698                         int burst_words, words;
 699                         u32 edm;
 700
 701                         burst_words = SDDATA_FIFO_PIO_BURST;
 702                         if (burst_words > copy_words)
 703                                 burst_words = copy_words;
 704                         edm = bcm2835_sdhost_read(host, SDEDM);
 705                         words = SDDATA_FIFO_WORDS - ((edm >> 4) & 0x1f);
 706
 707                         if (words < burst_words) {
 708                                 int fsm_state = (edm & SDEDM_FSM_MASK);
 709                                 if ((fsm_state != SDEDM_FSM_WRITEDATA) &&
 710                                     (fsm_state != SDEDM_FSM_WRITESTART1) &&
 711                                     (fsm_state != SDEDM_FSM_WRITESTART2)) {
 712                                         hsts = bcm2835_sdhost_read(host,
 713                                                                    SDHSTS);
 714                                         pr_info("%s: fsm %x, hsts %x\n",
 715                                                mmc_hostname(host->mmc),
 716                                                fsm_state, hsts);
 717                                         if (hsts & SDHSTS_ERROR_MASK)
 718                                                 break;
 719                                 }
 720
 721                                 if (time_after(jiffies, wait_max)) {
 722                                         pr_err("%s: PIO write timeout - EDM %x\n",
 723                                                mmc_hostname(host->mmc),
 724                                                edm);
 725                                         hsts = SDHSTS_REW_TIME_OUT;
 726                                         break;
 727                                 }
 728                                 ndelay((burst_words - words) *
 729                                        host->ns_per_fifo_word);
 730                                 continue;
 731                         } else if (words > copy_words) {
 732                                 words = copy_words;
 733                         }
 734
 735                         copy_words -= words;
 736
 737                         while (words) {
 738                                 bcm2835_sdhost_write(host, *(buf++), SDDATA);
 739                                 words--;
 740                         }
 741                 }
 742
 743                 if (hsts & SDHSTS_ERROR_MASK)
 744                         break;
 745         }
 746
 747         sg_miter_stop(&host->sg_miter);
 748
 749         local_irq_restore(flags);
 750 }
 751
 752 static void bcm2835_sdhost_transfer_pio(struct bcm2835_host *host)
 753 {
 754         u32 sdhsts;
 755         bool is_read;
 756         BUG_ON(!host->data);
 757         log_event("XFP<", host->data, host->blocks);
 758
 759         is_read = (host->data->flags & MMC_DATA_READ) != 0;
 760         if (is_read)
 761                 bcm2835_sdhost_read_block_pio(host);
 762         else
 763                 bcm2835_sdhost_write_block_pio(host);
 764
 765         sdhsts = bcm2835_sdhost_read(host, SDHSTS);
 766         if (sdhsts & (SDHSTS_CRC16_ERROR |
 767                       SDHSTS_CRC7_ERROR |
 768                       SDHSTS_FIFO_ERROR)) {
 769                 pr_err("%s: %s transfer error - HSTS %x\n",
 770                        mmc_hostname(host->mmc),
 771                        is_read ? "read" : "write",
 772                        sdhsts);
 773                 host->data->error = -EILSEQ;
 774         } else if ((sdhsts & (SDHSTS_CMD_TIME_OUT |
 775                               SDHSTS_REW_TIME_OUT))) {
 776                 pr_err("%s: %s timeout error - HSTS %x\n",
 777                        mmc_hostname(host->mmc),
 778                        is_read ? "read" : "write",
 779                        sdhsts);
 780                 host->data->error = -ETIMEDOUT;
 781         }
 782         log_event("XFP>", host->data, host->blocks);
 783 }
 784
 785 static void bcm2835_sdhost_prepare_dma(struct bcm2835_host *host,
 786         struct mmc_data *data)
 787 {
 788         int len, dir_data, dir_slave;
 789         struct dma_async_tx_descriptor *desc = NULL;
 790         struct dma_chan *dma_chan;
 791
 792         log_event("PRD<", data, 0);
 793         pr_debug("bcm2835_sdhost_prepare_dma()\n");
 794
 795         dma_chan = host->dma_chan_rxtx;
 796         if (data->flags & MMC_DATA_READ) {
 797                 dir_data = DMA_FROM_DEVICE;
 798                 dir_slave = DMA_DEV_TO_MEM;
 799         } else {
 800                 dir_data = DMA_TO_DEVICE;
 801                 dir_slave = DMA_MEM_TO_DEV;
 802         }
 803         log_event("PRD1", dma_chan, 0);
 804
 805         BUG_ON(!dma_chan->device);
 806         BUG_ON(!dma_chan->device->dev);
 807         BUG_ON(!data->sg);
 808
 809         /* The block doesn't manage the FIFO DREQs properly for multi-block
 810            transfers, so don't attempt to DMA the final few words.
 811            Unfortunately this requires the final sg entry to be trimmed.
 812            N.B. This code demands that the overspill is contained in
 813            a single sg entry.
 814         */
 815
 816         host->drain_words = 0;
 817         if ((data->blocks > 1) && (dir_data == DMA_FROM_DEVICE)) {
 818                 struct scatterlist *sg;
 819                 u32 len;
 820                 int i;
 821
 822                 len = min((u32)(FIFO_READ_THRESHOLD - 1) * 4,
 823                           (u32)data->blocks * data->blksz);
 824
 825                 for_each_sg(data->sg, sg, data->sg_len, i) {
 826                         if (sg_is_last(sg)) {
 827                                 BUG_ON(sg->length < len);
 828                                 sg->length -= len;
 829                                 host->drain_page = sg_page(sg);
 830                                 host->drain_offset = sg->offset + sg->length;
 831                         }
 832                 }
 833                 host->drain_words = len/4;
 834         }
 835
 836         /* The parameters have already been validated, so this will not fail */
 837         (void)dmaengine_slave_config(dma_chan,
 838                                      (dir_data == DMA_FROM_DEVICE) ?
 839                                      &host->dma_cfg_rx :
 840                                      &host->dma_cfg_tx);
 841
 842         len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
 843                          dir_data);
 844
 845         log_event("PRD2", len, 0);
 846         if (len > 0)
 847                 desc = dmaengine_prep_slave_sg(dma_chan, data->sg,
 848                                                len, dir_slave,
 849                                                DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 850         log_event("PRD3", desc, 0);
 851
 852         if (desc) {
 853                 desc->callback = bcm2835_sdhost_dma_complete;
 854                 desc->callback_param = host;
 855                 host->dma_desc = desc;
 856                 host->dma_chan = dma_chan;
 857                 host->dma_dir = dir_data;
 858         }
 859         log_event("PDM>", data, 0);
 860 }
 861
 862 static void bcm2835_sdhost_start_dma(struct bcm2835_host *host)
 863 {
 864         log_event("SDMA", host->data, host->dma_chan);
 865         dmaengine_submit(host->dma_desc);
 866         dma_async_issue_pending(host->dma_chan);
 867 }
 868
 869 static void bcm2835_sdhost_set_transfer_irqs(struct bcm2835_host *host)
 870 {
 871         u32 all_irqs = SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN |
 872                 SDHCFG_BUSY_IRPT_EN;
 873         if (host->dma_desc)
 874                 host->hcfg = (host->hcfg & ~all_irqs) |
 875                         SDHCFG_BUSY_IRPT_EN;
 876         else
 877                 host->hcfg = (host->hcfg & ~all_irqs) |
 878                         SDHCFG_DATA_IRPT_EN |
 879                         SDHCFG_BUSY_IRPT_EN;
 880
 881         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 882 }
 883
 884 static void bcm2835_sdhost_prepare_data(struct bcm2835_host *host, struct mmc_command *cmd)
 885 {
 886         struct mmc_data *data = cmd->data;
 887
 888         WARN_ON(host->data);
 889
 890         host->data = data;
 891         if (!data)
 892                 return;
 893
 894         /* Sanity checks */
 895         BUG_ON(data->blksz * data->blocks > 524288);
 896         BUG_ON(data->blksz > host->mmc->max_blk_size);
 897         BUG_ON(data->blocks > 65535);
 898
 899         host->data_complete = 0;
 900         host->flush_fifo = 0;
 901         host->data->bytes_xfered = 0;
 902
 903         if (!host->sectors && host->mmc->card) {
 904                 struct mmc_card *card = host->mmc->card;
 905                 if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
 906                         /*
 907                          * The EXT_CSD sector count is in number of 512 byte
 908                          * sectors.
 909                          */
 910                         host->sectors = card->ext_csd.sectors;
 911                 } else {
 912                         /*
 913                          * The CSD capacity field is in units of read_blkbits.
 914                          * set_capacity takes units of 512 bytes.
 915                          */
 916                         host->sectors = card->csd.capacity <<
 917                                 (card->csd.read_blkbits - 9);
 918                 }
 919         }
 920
 921         if (!host->dma_desc) {
 922                 /* Use PIO */
 923                 int flags = SG_MITER_ATOMIC;
 924
 925                 if (data->flags & MMC_DATA_READ)
 926                         flags |= SG_MITER_TO_SG;
 927                 else
 928                         flags |= SG_MITER_FROM_SG;
 929                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 930                 host->blocks = data->blocks;
 931         }
 932
 933         bcm2835_sdhost_set_transfer_irqs(host);
 934
 935         bcm2835_sdhost_write(host, data->blksz, SDHBCT);
 936         bcm2835_sdhost_write(host, data->blocks, SDHBLC);
 937
 938         BUG_ON(!host->data);
 939 }
 940
 941 bool bcm2835_sdhost_send_command(struct bcm2835_host *host,
 942                                  struct mmc_command *cmd)
 943 {
 944         u32 sdcmd, sdhsts;
 945         unsigned long timeout;
 946         int delay;
 947
 948         WARN_ON(host->cmd);
 949         log_event("CMD<", cmd->opcode, cmd->arg);
 950
 951         if (cmd->data)
 952                 pr_debug("%s: send_command %d 0x%x "
 953                          "(flags 0x%x) - %s %d*%d\n",
 954                          mmc_hostname(host->mmc),
 955                          cmd->opcode, cmd->arg, cmd->flags,
 956                          (cmd->data->flags & MMC_DATA_READ) ?
 957                          "read" : "write", cmd->data->blocks,
 958                          cmd->data->blksz);
 959         else
 960                 pr_debug("%s: send_command %d 0x%x (flags 0x%x)\n",
 961                          mmc_hostname(host->mmc),
 962                          cmd->opcode, cmd->arg, cmd->flags);
 963
 964         /* Wait max 100 ms */
 965         timeout = 10000;
 966
 967         while (bcm2835_sdhost_read(host, SDCMD) & SDCMD_NEW_FLAG) {
 968                 if (timeout == 0) {
 969                         pr_warn("%s: previous command never completed.\n",
 970                                 mmc_hostname(host->mmc));
 971                         if (host->debug)
 972                                 bcm2835_sdhost_dumpregs(host);
 973                         cmd->error = -EILSEQ;
 974                         tasklet_schedule(&host->finish_tasklet);
 975                         return false;
 976                 }
 977                 timeout--;
 978                 udelay(10);
 979         }
 980
 981         delay = (10000 - timeout)/100;
 982         if (delay > host->max_delay) {
 983                 host->max_delay = delay;
 984                 pr_warn("%s: controller hung for %d ms\n",
 985                            mmc_hostname(host->mmc),
 986                            host->max_delay);
 987         }
 988
 989         timeout = jiffies;
 990         if (!cmd->data && cmd->busy_timeout > 9000)
 991                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
 992         else
 993                 timeout += 10 * HZ;
 994         mod_timer(&host->timer, timeout);
 995
 996         host->cmd = cmd;
 997
 998         /* Clear any error flags */
 999         sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1000         if (sdhsts & SDHSTS_ERROR_MASK)
1001                 bcm2835_sdhost_write(host, sdhsts, SDHSTS);
1002
1003         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1004                 pr_err("%s: unsupported response type!\n",
1005                         mmc_hostname(host->mmc));
1006                 cmd->error = -EINVAL;
1007                 tasklet_schedule(&host->finish_tasklet);
1008                 return false;
1009         }
1010
1011         bcm2835_sdhost_prepare_data(host, cmd);
1012
1013         bcm2835_sdhost_write(host, cmd->arg, SDARG);
1014
1015         sdcmd = cmd->opcode & SDCMD_CMD_MASK;
1016
1017         host->use_busy = 0;
1018         if (!(cmd->flags & MMC_RSP_PRESENT)) {
1019                 sdcmd |= SDCMD_NO_RESPONSE;
1020         } else {
1021                 if (cmd->flags & MMC_RSP_136)
1022                         sdcmd |= SDCMD_LONG_RESPONSE;
1023                 if (cmd->flags & MMC_RSP_BUSY) {
1024                         sdcmd |= SDCMD_BUSYWAIT;
1025                         host->use_busy = 1;
1026                 }
1027         }
1028
1029         if (cmd->data) {
1030                 log_event("CMDD", cmd->data->blocks, cmd->data->blksz);
1031                 if (host->delay_after_this_stop) {
1032                         struct timespec64 now;
1033                         int time_since_stop;
1034
1035                         ktime_get_real_ts64(&now);
1036                         time_since_stop = now.tv_sec - host->stop_time.tv_sec;
1037                         if (time_since_stop < 2) {
1038                                 /* Possibly less than one second */
1039                                 time_since_stop = time_since_stop * 1000000 +
1040                                         (now.tv_nsec - host->stop_time.tv_nsec)/1000;
1041                                 if (time_since_stop <
1042                                     host->delay_after_this_stop)
1043                                         udelay(host->delay_after_this_stop -
1044                                                time_since_stop);
1045                         }
1046                 }
1047
1048                 host->delay_after_this_stop = host->delay_after_stop;
1049                 if ((cmd->data->flags & MMC_DATA_READ) && !host->use_sbc) {
1050                         /* See if read crosses one of the hazardous sectors */
1051                         u32 first_blk, last_blk;
1052
1053                         /* Intentionally include the following sector because
1054                            without CMD23/SBC the read may run on. */
1055                         first_blk = host->mrq->cmd->arg;
1056                         last_blk = first_blk + cmd->data->blocks;
1057
1058                         if (((last_blk >= (host->sectors - 64)) &&
1059                              (first_blk <= (host->sectors - 64))) ||
1060                             ((last_blk >= (host->sectors - 32)) &&
1061                              (first_blk <= (host->sectors - 32)))) {
1062                                 host->delay_after_this_stop =
1063                                         max(250u, host->delay_after_stop);
1064                         }
1065                 }
1066
1067                 if (cmd->data->flags & MMC_DATA_WRITE)
1068                         sdcmd |= SDCMD_WRITE_CMD;
1069                 if (cmd->data->flags & MMC_DATA_READ)
1070                         sdcmd |= SDCMD_READ_CMD;
1071         }
1072
1073         bcm2835_sdhost_write(host, sdcmd | SDCMD_NEW_FLAG, SDCMD);
1074
1075         return true;
1076 }
1077
1078 static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1079                                           unsigned long *irq_flags);
1080 static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host);
1081
1082 static void bcm2835_sdhost_finish_data(struct bcm2835_host *host)
1083 {
1084         struct mmc_data *data;
1085
1086         data = host->data;
1087         BUG_ON(!data);
1088
1089         log_event("FDA<", host->mrq, host->cmd);
1090         pr_debug("finish_data(error %d, stop %d, sbc %d)\n",
1091                data->error, data->stop ? 1 : 0,
1092                host->mrq->sbc ? 1 : 0);
1093
1094         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
1095         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1096
1097         data->bytes_xfered = data->error ? 0 : (data->blksz * data->blocks);
1098
1099         host->data_complete = 1;
1100
1101         if (host->cmd) {
1102                 /*
1103                  * Data managed to finish before the
1104                  * command completed. Make sure we do
1105                  * things in the proper order.
1106                  */
1107                 pr_debug("Finished early - HSTS %x\n",
1108                          bcm2835_sdhost_read(host, SDHSTS));
1109         }
1110         else
1111                 bcm2835_sdhost_transfer_complete(host);
1112         log_event("FDA>", host->mrq, host->cmd);
1113 }
1114
1115 static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host)
1116 {
1117         struct mmc_data *data;
1118
1119         BUG_ON(host->cmd);
1120         BUG_ON(!host->data);
1121         BUG_ON(!host->data_complete);
1122
1123         data = host->data;
1124         host->data = NULL;
1125
1126         log_event("TCM<", data, data->error);
1127         pr_debug("transfer_complete(error %d, stop %d)\n",
1128                data->error, data->stop ? 1 : 0);
1129
1130         /*
1131          * Need to send CMD12 if -
1132          * a) open-ended multiblock transfer (no CMD23)
1133          * b) error in multiblock transfer
1134          */
1135         if (host->mrq->stop && (data->error || !host->use_sbc)) {
1136                 if (bcm2835_sdhost_send_command(host, host->mrq->stop)) {
1137                         /* No busy, so poll for completion */
1138                         if (!host->use_busy)
1139                                 bcm2835_sdhost_finish_command(host, NULL);
1140
1141                         if (host->delay_after_this_stop)
1142                                 ktime_get_real_ts64(&host->stop_time);
1143                 }
1144         } else {
1145                 bcm2835_sdhost_wait_transfer_complete(host);
1146                 tasklet_schedule(&host->finish_tasklet);
1147         }
1148         log_event("TCM>", data, 0);
1149 }
1150
1151 /* If irq_flags is valid, the caller is in a thread context and is allowed
1152    to sleep */
1153 static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1154                                           unsigned long *irq_flags)
1155 {
1156         u32 sdcmd;
1157         u32 retries;
1158 #ifdef DEBUG
1159         struct timespec64 before, after;
1160         int timediff = 0;
1161 #endif
1162
1163         log_event("FCM<", host->mrq, host->cmd);
1164         pr_debug("finish_command(%x)\n", bcm2835_sdhost_read(host, SDCMD));
1165
1166         BUG_ON(!host->cmd || !host->mrq);
1167
1168         /* Poll quickly at first */
1169
1170         retries = host->cmd_quick_poll_retries;
1171         if (!retries) {
1172                 /* Work out how many polls take 1us by timing 10us */
1173                 struct timespec64 start, now;
1174                 int us_diff;
1175
1176                 retries = 1;
1177                 do {
1178                         int i;
1179
1180                         retries *= 2;
1181
1182                         ktime_get_real_ts64(&start);
1183
1184                         for (i = 0; i < retries; i++) {
1185                                 cpu_relax();
1186                                 sdcmd = bcm2835_sdhost_read(host, SDCMD);
1187                         }
1188
1189                         ktime_get_real_ts64(&now);
1190                         us_diff = (now.tv_sec - start.tv_sec) * 1000000 +
1191                                 (now.tv_nsec - start.tv_nsec)/1000;
1192                 } while (us_diff < 10);
1193
1194                 host->cmd_quick_poll_retries = ((retries * us_diff + 9)*CMD_DALLY_US)/10 + 1;
1195                 retries = 1; // We've already waited long enough this time
1196         }
1197
1198         for (sdcmd = bcm2835_sdhost_read(host, SDCMD);
1199              (sdcmd & SDCMD_NEW_FLAG) && retries;
1200              retries--) {
1201                 cpu_relax();
1202                 sdcmd = bcm2835_sdhost_read(host, SDCMD);
1203         }
1204
1205         if (!retries) {
1206                 unsigned long wait_max;
1207
1208                 if (!irq_flags) {
1209                         /* Schedule the work */
1210                         log_event("CWWQ", 0, 0);
1211                         schedule_work(&host->cmd_wait_wq);
1212                         return;
1213                 }
1214
1215                 /* Wait max 100 ms */
1216                 wait_max = jiffies + msecs_to_jiffies(100);
1217                 while (time_before(jiffies, wait_max)) {
1218                         spin_unlock_irqrestore(&host->lock, *irq_flags);
1219                         usleep_range(1, 10);
1220                         spin_lock_irqsave(&host->lock, *irq_flags);
1221                         sdcmd = bcm2835_sdhost_read(host, SDCMD);
1222                         if (!(sdcmd & SDCMD_NEW_FLAG))
1223                                 break;
1224                 }
1225         }
1226
1227         /* Check for errors */
1228         if (sdcmd & SDCMD_NEW_FLAG) {
1229                 if (host->debug) {
1230                         pr_err("%s: command %d never completed.\n",
1231                                mmc_hostname(host->mmc), host->cmd->opcode);
1232                         bcm2835_sdhost_dumpregs(host);
1233                 }
1234                 host->cmd->error = -EILSEQ;
1235                 tasklet_schedule(&host->finish_tasklet);
1236                 return;
1237         } else if (sdcmd & SDCMD_FAIL_FLAG) {
1238                 u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1239
1240                 /* Clear the errors */
1241                 bcm2835_sdhost_write(host, SDHSTS_ERROR_MASK, SDHSTS);
1242
1243                 if (host->debug)
1244                         pr_info("%s: error detected - CMD %x, HSTS %03x, EDM %x\n",
1245                                 mmc_hostname(host->mmc), sdcmd, sdhsts,
1246                                 bcm2835_sdhost_read(host, SDEDM));
1247
1248                 if ((sdhsts & SDHSTS_CRC7_ERROR) &&
1249                     (host->cmd->opcode == 1)) {
1250                         if (host->debug)
1251                                 pr_info("%s: ignoring CRC7 error for CMD1\n",
1252                                         mmc_hostname(host->mmc));
1253                 } else {
1254                         u32 edm, fsm;
1255
1256                         if (sdhsts & SDHSTS_CMD_TIME_OUT) {
1257                                 if (host->debug)
1258                                         pr_warn("%s: command %d timeout\n",
1259                                                mmc_hostname(host->mmc),
1260                                                host->cmd->opcode);
1261                                 host->cmd->error = -ETIMEDOUT;
1262                         } else {
1263                                 pr_warn("%s: unexpected command %d error\n",
1264                                        mmc_hostname(host->mmc),
1265                                        host->cmd->opcode);
1266                                 host->cmd->error = -EILSEQ;
1267                         }
1268
1269                         edm = readl(host->ioaddr + SDEDM);
1270                         fsm = edm & SDEDM_FSM_MASK;
1271                         if (fsm == SDEDM_FSM_READWAIT ||
1272                             fsm == SDEDM_FSM_WRITESTART1)
1273                                 writel(edm | SDEDM_FORCE_DATA_MODE,
1274                                        host->ioaddr + SDEDM);
1275                         tasklet_schedule(&host->finish_tasklet);
1276                         return;
1277                 }
1278         }
1279
1280         if (host->cmd->flags & MMC_RSP_PRESENT) {
1281                 if (host->cmd->flags & MMC_RSP_136) {
1282                         int i;
1283                         for (i = 0; i < 4; i++)
1284                                 host->cmd->resp[3 - i] = bcm2835_sdhost_read(host, SDRSP0 + i*4);
1285                         pr_debug("%s: finish_command %08x %08x %08x %08x\n",
1286                                  mmc_hostname(host->mmc),
1287                                  host->cmd->resp[0], host->cmd->resp[1], host->cmd->resp[2], host->cmd->resp[3]);
1288                         log_event("RSP ", host->cmd->resp[0], host->cmd->resp[1]);
1289                 } else {
1290                         host->cmd->resp[0] = bcm2835_sdhost_read(host, SDRSP0);
1291                         pr_debug("%s: finish_command %08x\n",
1292                                  mmc_hostname(host->mmc),
1293                                  host->cmd->resp[0]);
1294                         log_event("RSP ", host->cmd->resp[0], 0);
1295                 }
1296         }
1297
1298         if (host->cmd == host->mrq->sbc) {
1299                 /* Finished CMD23, now send actual command. */
1300                 host->cmd = NULL;
1301                 if (bcm2835_sdhost_send_command(host, host->mrq->cmd)) {
1302                         if (host->data && host->dma_desc)
1303                                 /* DMA transfer starts now, PIO starts after irq */
1304                                 bcm2835_sdhost_start_dma(host);
1305
1306                         if (!host->use_busy)
1307                                 bcm2835_sdhost_finish_command(host, NULL);
1308                 }
1309         } else if (host->cmd == host->mrq->stop) {
1310                 /* Finished CMD12 */
1311                 tasklet_schedule(&host->finish_tasklet);
1312         } else {
1313                 /* Processed actual command. */
1314                 host->cmd = NULL;
1315                 if (!host->data)
1316                         tasklet_schedule(&host->finish_tasklet);
1317                 else if (host->data_complete)
1318                         bcm2835_sdhost_transfer_complete(host);
1319         }
1320         log_event("FCM>", host->mrq, host->cmd);
1321 }
1322
1323 static void bcm2835_sdhost_timeout(struct timer_list *t)
1324 {
1325         struct bcm2835_host *host = from_timer(host, t, timer);
1326         unsigned long flags;
1327
1328         spin_lock_irqsave(&host->lock, flags);
1329         log_event("TIM<", 0, 0);
1330
1331         if (host->mrq) {
1332                 pr_err("%s: timeout waiting for hardware interrupt.\n",
1333                         mmc_hostname(host->mmc));
1334                 log_dump();
1335                 bcm2835_sdhost_dumpregs(host);
1336
1337                 if (host->data) {
1338                         host->data->error = -ETIMEDOUT;
1339                         bcm2835_sdhost_finish_data(host);
1340                 } else {
1341                         if (host->cmd)
1342                                 host->cmd->error = -ETIMEDOUT;
1343                         else
1344                                 host->mrq->cmd->error = -ETIMEDOUT;
1345
1346                         pr_debug("timeout_timer tasklet_schedule\n");
1347                         tasklet_schedule(&host->finish_tasklet);
1348                 }
1349         }
1350
1351         spin_unlock_irqrestore(&host->lock, flags);
1352 }
1353
1354 static void bcm2835_sdhost_busy_irq(struct bcm2835_host *host, u32 intmask)
1355 {
1356         log_event("IRQB", host->cmd, intmask);
1357         if (!host->cmd) {
1358                 pr_err("%s: got command busy interrupt 0x%08x even "
1359                         "though no command operation was in progress.\n",
1360                         mmc_hostname(host->mmc), (unsigned)intmask);
1361                 bcm2835_sdhost_dumpregs(host);
1362                 return;
1363         }
1364
1365         if (!host->use_busy) {
1366                 pr_err("%s: got command busy interrupt 0x%08x even "
1367                         "though not expecting one.\n",
1368                         mmc_hostname(host->mmc), (unsigned)intmask);
1369                 bcm2835_sdhost_dumpregs(host);
1370                 return;
1371         }
1372         host->use_busy = 0;
1373
1374         if (intmask & SDHSTS_ERROR_MASK)
1375         {
1376                 pr_err("sdhost_busy_irq: intmask %x, data %p\n", intmask, host->mrq->data);
1377                 if (intmask & SDHSTS_CRC7_ERROR)
1378                         host->cmd->error = -EILSEQ;
1379                 else if (intmask & (SDHSTS_CRC16_ERROR |
1380                                     SDHSTS_FIFO_ERROR)) {
1381                         if (host->mrq->data)
1382                                 host->mrq->data->error = -EILSEQ;
1383                         else
1384                                 host->cmd->error = -EILSEQ;
1385                 } else if (intmask & SDHSTS_REW_TIME_OUT) {
1386                         if (host->mrq->data)
1387                                 host->mrq->data->error = -ETIMEDOUT;
1388                         else
1389                                 host->cmd->error = -ETIMEDOUT;
1390                 } else if (intmask & SDHSTS_CMD_TIME_OUT)
1391                         host->cmd->error = -ETIMEDOUT;
1392
1393                 if (host->debug) {
1394                         log_dump();
1395                         bcm2835_sdhost_dumpregs(host);
1396                 }
1397         }
1398         else
1399                 bcm2835_sdhost_finish_command(host, NULL);
1400 }
1401
1402 static void bcm2835_sdhost_data_irq(struct bcm2835_host *host, u32 intmask)
1403 {
1404         /* There are no dedicated data/space available interrupt
1405            status bits, so it is necessary to use the single shared
1406            data/space available FIFO status bits. It is therefore not
1407            an error to get here when there is no data transfer in
1408            progress. */
1409         log_event("IRQD", host->data, intmask);
1410         if (!host->data)
1411                 return;
1412
1413         if (intmask & (SDHSTS_CRC16_ERROR |
1414                        SDHSTS_FIFO_ERROR |
1415                        SDHSTS_REW_TIME_OUT)) {
1416                 if (intmask & (SDHSTS_CRC16_ERROR |
1417                                SDHSTS_FIFO_ERROR))
1418                         host->data->error = -EILSEQ;
1419                 else
1420                         host->data->error = -ETIMEDOUT;
1421
1422                 if (host->debug) {
1423                         log_dump();
1424                         bcm2835_sdhost_dumpregs(host);
1425                 }
1426         }
1427
1428         if (host->data->error) {
1429                 bcm2835_sdhost_finish_data(host);
1430         } else if (host->data->flags & MMC_DATA_WRITE) {
1431                 /* Use the block interrupt for writes after the first block */
1432                 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN);
1433                 host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
1434                 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1435                 bcm2835_sdhost_transfer_pio(host);
1436         } else {
1437                 bcm2835_sdhost_transfer_pio(host);
1438                 host->blocks--;
1439                 if ((host->blocks == 0) || host->data->error)
1440                         bcm2835_sdhost_finish_data(host);
1441         }
1442 }
1443
1444 static void bcm2835_sdhost_block_irq(struct bcm2835_host *host, u32 intmask)
1445 {
1446         log_event("IRQK", host->data, intmask);
1447         if (!host->data) {
1448                 pr_err("%s: got block interrupt 0x%08x even "
1449                         "though no data operation was in progress.\n",
1450                         mmc_hostname(host->mmc), (unsigned)intmask);
1451                 bcm2835_sdhost_dumpregs(host);
1452                 return;
1453         }
1454
1455         if (intmask & (SDHSTS_CRC16_ERROR |
1456                        SDHSTS_FIFO_ERROR |
1457                        SDHSTS_REW_TIME_OUT)) {
1458                 if (intmask & (SDHSTS_CRC16_ERROR |
1459                                SDHSTS_FIFO_ERROR))
1460                         host->data->error = -EILSEQ;
1461                 else
1462                         host->data->error = -ETIMEDOUT;
1463
1464                 if (host->debug) {
1465                         log_dump();
1466                         bcm2835_sdhost_dumpregs(host);
1467                 }
1468         }
1469
1470         if (!host->dma_desc) {
1471                 BUG_ON(!host->blocks);
1472                 if (host->data->error || (--host->blocks == 0)) {
1473                         bcm2835_sdhost_finish_data(host);
1474                 } else {
1475                         bcm2835_sdhost_transfer_pio(host);
1476                 }
1477         } else if (host->data->flags & MMC_DATA_WRITE) {
1478                 bcm2835_sdhost_finish_data(host);
1479         }
1480 }
1481
1482 static irqreturn_t bcm2835_sdhost_irq(int irq, void *dev_id)
1483 {
1484         irqreturn_t result = IRQ_NONE;
1485         struct bcm2835_host *host = dev_id;
1486         u32 intmask;
1487
1488         spin_lock(&host->lock);
1489
1490         intmask = bcm2835_sdhost_read(host, SDHSTS);
1491         log_event("IRQ<", intmask, 0);
1492
1493         bcm2835_sdhost_write(host,
1494                              SDHSTS_BUSY_IRPT |
1495                              SDHSTS_BLOCK_IRPT |
1496                              SDHSTS_SDIO_IRPT |
1497                              SDHSTS_DATA_FLAG,
1498                              SDHSTS);
1499
1500         if (intmask & SDHSTS_BLOCK_IRPT) {
1501                 bcm2835_sdhost_block_irq(host, intmask);
1502                 result = IRQ_HANDLED;
1503         }
1504
1505         if (intmask & SDHSTS_BUSY_IRPT) {
1506                 bcm2835_sdhost_busy_irq(host, intmask);
1507                 result = IRQ_HANDLED;
1508         }
1509
1510         /* There is no true data interrupt status bit, so it is
1511            necessary to qualify the data flag with the interrupt
1512            enable bit */
1513         if ((intmask & SDHSTS_DATA_FLAG) &&
1514             (host->hcfg & SDHCFG_DATA_IRPT_EN)) {
1515                 bcm2835_sdhost_data_irq(host, intmask);
1516                 result = IRQ_HANDLED;
1517         }
1518
1519         log_event("IRQ>", bcm2835_sdhost_read(host, SDHSTS), 0);
1520         spin_unlock(&host->lock);
1521
1522         return result;
1523 }
1524
1525 void bcm2835_sdhost_set_clock(struct bcm2835_host *host, unsigned int clock)
1526 {
1527         int div = 0; /* Initialized for compiler warning */
1528         unsigned int input_clock = clock;
1529         unsigned long flags;
1530
1531         if (host->debug)
1532                 pr_info("%s: set_clock(%d)\n", mmc_hostname(host->mmc), clock);
1533
1534         if (host->overclock_50 && (clock == 50*MHZ))
1535                 clock = host->overclock_50 * MHZ + (MHZ - 1);
1536
1537         /* The SDCDIV register has 11 bits, and holds (div - 2).
1538            But in data mode the max is 50MHz wihout a minimum, and only the
1539            bottom 3 bits are used. Since the switch over is automatic (unless
1540            we have marked the card as slow...), chosen values have to make
1541            sense in both modes.
1542            Ident mode must be 100-400KHz, so can range check the requested
1543            clock. CMD15 must be used to return to data mode, so this can be
1544            monitored.
1545
1546            clock 250MHz -> 0->125MHz, 1->83.3MHz, 2->62.5MHz, 3->50.0MHz
1547                            4->41.7MHz, 5->35.7MHz, 6->31.3MHz, 7->27.8MHz
1548
1549                          623->400KHz/27.8MHz
1550                          reset value (507)->491159/50MHz
1551
1552            BUT, the 3-bit clock divisor in data mode is too small if the
1553            core clock is higher than 250MHz, so instead use the SLOW_CARD
1554            configuration bit to force the use of the ident clock divisor
1555            at all times.
1556         */
1557
1558         host->mmc->actual_clock = 0;
1559
1560         if (host->firmware_sets_cdiv) {
1561                 u32 msg[3] = { clock, 0, 0 };
1562
1563                 rpi_firmware_property(host->fw,
1564                                       RPI_FIRMWARE_SET_SDHOST_CLOCK,
1565                                       &msg, sizeof(msg));
1566
1567                 clock = max(msg[1], msg[2]);
1568                 spin_lock_irqsave(&host->lock, flags);
1569         } else {
1570                 spin_lock_irqsave(&host->lock, flags);
1571                 if (clock < 100000) {
1572                         /* Can't stop the clock, but make it as slow as
1573                          * possible to show willing
1574                          */
1575                         host->cdiv = SDCDIV_MAX_CDIV;
1576                         bcm2835_sdhost_write(host, host->cdiv, SDCDIV);
1577                         spin_unlock_irqrestore(&host->lock, flags);
1578                         return;
1579                 }
1580
1581                 div = host->max_clk / clock;
1582                 if (div < 2)
1583                         div = 2;
1584                 if ((host->max_clk / div) > clock)
1585                         div++;
1586                 div -= 2;
1587
1588                 if (div > SDCDIV_MAX_CDIV)
1589                         div = SDCDIV_MAX_CDIV;
1590
1591                 clock = host->max_clk / (div + 2);
1592
1593                 host->cdiv = div;
1594                 bcm2835_sdhost_write(host, host->cdiv, SDCDIV);
1595
1596                 if (host->debug)
1597                         pr_info("%s: clock=%d -> max_clk=%d, cdiv=%x "
1598                                 "(actual clock %d)\n",
1599                                 mmc_hostname(host->mmc), input_clock,
1600                                 host->max_clk, host->cdiv,
1601                                 clock);
1602         }
1603
1604         /* Calibrate some delays */
1605
1606         host->ns_per_fifo_word = (1000000000/clock) *
1607                 ((host->mmc->caps & MMC_CAP_4_BIT_DATA) ? 8 : 32);
1608
1609         if (input_clock == 50 * MHZ) {
1610                 if (clock > input_clock) {
1611                         /* Save the closest value, to make it easier
1612                            to reduce in the event of error */
1613                         host->overclock_50 = (clock/MHZ);
1614
1615                         if (clock != host->overclock) {
1616                                 pr_info("%s: overclocking to %dHz\n",
1617                                         mmc_hostname(host->mmc), clock);
1618                                 host->overclock = clock;
1619                         }
1620                 } else if (host->overclock) {
1621                         host->overclock = 0;
1622                         if (clock == 50 * MHZ)
1623                                 pr_warn("%s: cancelling overclock\n",
1624                                         mmc_hostname(host->mmc));
1625                 }
1626         } else if (input_clock == 0) {
1627                 /* Reset the preferred overclock when the clock is stopped.
1628                  * This always happens during initialisation. */
1629                 host->overclock_50 = host->user_overclock_50;
1630                 host->overclock = 0;
1631         }
1632
1633         /* Set the timeout to 500ms */
1634         bcm2835_sdhost_write(host, clock/2, SDTOUT);
1635
1636         host->mmc->actual_clock = clock;
1637         host->clock = input_clock;
1638         host->reset_clock = 0;
1639
1640         spin_unlock_irqrestore(&host->lock, flags);
1641 }
1642
1643 static void bcm2835_sdhost_request(struct mmc_host *mmc, struct mmc_request *mrq)
1644 {
1645         struct bcm2835_host *host;
1646         unsigned long flags;
1647         u32 edm, fsm;
1648
1649         host = mmc_priv(mmc);
1650
1651         if (host->debug) {
1652                 struct mmc_command *cmd = mrq->cmd;
1653                 BUG_ON(!cmd);
1654                 if (cmd->data)
1655                         pr_info("%s: cmd %d 0x%x (flags 0x%x) - %s %d*%d\n",
1656                                 mmc_hostname(mmc),
1657                                 cmd->opcode, cmd->arg, cmd->flags,
1658                                 (cmd->data->flags & MMC_DATA_READ) ?
1659                                 "read" : "write", cmd->data->blocks,
1660                                 cmd->data->blksz);
1661                 else
1662                         pr_info("%s: cmd %d 0x%x (flags 0x%x)\n",
1663                                 mmc_hostname(mmc),
1664                                 cmd->opcode, cmd->arg, cmd->flags);
1665         }
1666
1667         /* Reset the error statuses in case this is a retry */
1668         if (mrq->sbc)
1669                 mrq->sbc->error = 0;
1670         if (mrq->cmd)
1671                 mrq->cmd->error = 0;
1672         if (mrq->data)
1673                 mrq->data->error = 0;
1674         if (mrq->stop)
1675                 mrq->stop->error = 0;
1676
1677         if (mrq->data && !is_power_of_2(mrq->data->blksz)) {
1678                 pr_err("%s: unsupported block size (%d bytes)\n",
1679                        mmc_hostname(mmc), mrq->data->blksz);
1680                 mrq->cmd->error = -EINVAL;
1681                 mmc_request_done(mmc, mrq);
1682                 return;
1683         }
1684
1685         if (host->use_dma && mrq->data &&
1686             (mrq->data->blocks > host->pio_limit))
1687                 bcm2835_sdhost_prepare_dma(host, mrq->data);
1688
1689         if (host->reset_clock)
1690             bcm2835_sdhost_set_clock(host, host->clock);
1691
1692         spin_lock_irqsave(&host->lock, flags);
1693
1694         WARN_ON(host->mrq != NULL);
1695         host->mrq = mrq;
1696
1697         edm = bcm2835_sdhost_read(host, SDEDM);
1698         fsm = edm & SDEDM_FSM_MASK;
1699
1700         log_event("REQ<", mrq, edm);
1701         if ((fsm != SDEDM_FSM_IDENTMODE) &&
1702             (fsm != SDEDM_FSM_DATAMODE)) {
1703                 log_event("REQ!", mrq, edm);
1704                 if (host->debug) {
1705                         pr_warn("%s: previous command (%d) not complete (EDM %x)\n",
1706                                mmc_hostname(host->mmc),
1707                                bcm2835_sdhost_read(host, SDCMD) & SDCMD_CMD_MASK,
1708                                edm);
1709                         log_dump();
1710                         bcm2835_sdhost_dumpregs(host);
1711                 }
1712                 mrq->cmd->error = -EILSEQ;
1713                 tasklet_schedule(&host->finish_tasklet);
1714                 spin_unlock_irqrestore(&host->lock, flags);
1715                 return;
1716         }
1717
1718         host->use_sbc = !!mrq->sbc &&
1719                 (host->mrq->data->flags & USE_CMD23_FLAGS);
1720         if (host->use_sbc) {
1721                 if (bcm2835_sdhost_send_command(host, mrq->sbc)) {
1722                         if (!host->use_busy)
1723                                 bcm2835_sdhost_finish_command(host, &flags);
1724                 }
1725         } else if (bcm2835_sdhost_send_command(host, mrq->cmd)) {
1726                 if (host->data && host->dma_desc)
1727                         /* DMA transfer starts now, PIO starts after irq */
1728                         bcm2835_sdhost_start_dma(host);
1729
1730                 if (!host->use_busy)
1731                         bcm2835_sdhost_finish_command(host, &flags);
1732         }
1733
1734         log_event("CMD ", mrq->cmd->opcode,
1735                    mrq->data ? (u32)mrq->data->blksz : 0);
1736
1737         log_event("REQ>", mrq, 0);
1738         spin_unlock_irqrestore(&host->lock, flags);
1739 }
1740
1741 static void bcm2835_sdhost_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1742 {
1743
1744         struct bcm2835_host *host = mmc_priv(mmc);
1745         unsigned long flags;
1746
1747         if (host->debug)
1748                 pr_info("%s: ios clock %d, pwr %d, bus_width %d, "
1749                         "timing %d, vdd %d, drv_type %d\n",
1750                         mmc_hostname(mmc),
1751                         ios->clock, ios->power_mode, ios->bus_width,
1752                         ios->timing, ios->signal_voltage, ios->drv_type);
1753
1754         spin_lock_irqsave(&host->lock, flags);
1755
1756         log_event("IOS<", ios->clock, 0);
1757
1758         /* set bus width */
1759         host->hcfg &= ~SDHCFG_WIDE_EXT_BUS;
1760         if (ios->bus_width == MMC_BUS_WIDTH_4)
1761                 host->hcfg |= SDHCFG_WIDE_EXT_BUS;
1762
1763         host->hcfg |= SDHCFG_WIDE_INT_BUS;
1764
1765         /* Disable clever clock switching, to cope with fast core clocks */
1766         host->hcfg |= SDHCFG_SLOW_CARD;
1767
1768         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1769
1770         spin_unlock_irqrestore(&host->lock, flags);
1771
1772         if (!ios->clock || ios->clock != host->clock)
1773                 bcm2835_sdhost_set_clock(host, ios->clock);
1774 }
1775
1776 static struct mmc_host_ops bcm2835_sdhost_ops = {
1777         .request = bcm2835_sdhost_request,
1778         .set_ios = bcm2835_sdhost_set_ios,
1779         .hw_reset = bcm2835_sdhost_reset,
1780 };
1781
1782 static void bcm2835_sdhost_cmd_wait_work(struct work_struct *work)
1783 {
1784         struct bcm2835_host *host;
1785         unsigned long flags;
1786
1787         host = container_of(work, struct bcm2835_host, cmd_wait_wq);
1788
1789         spin_lock_irqsave(&host->lock, flags);
1790
1791         log_event("CWK<", host->cmd, host->mrq);
1792
1793         /*
1794          * If this tasklet gets rescheduled while running, it will
1795          * be run again afterwards but without any active request.
1796          */
1797         if (!host->mrq) {
1798                 spin_unlock_irqrestore(&host->lock, flags);
1799                 return;
1800         }
1801
1802         bcm2835_sdhost_finish_command(host, &flags);
1803
1804         log_event("CWK>", host->cmd, 0);
1805
1806         spin_unlock_irqrestore(&host->lock, flags);
1807 }
1808
1809 static void bcm2835_sdhost_tasklet_finish(unsigned long param)
1810 {
1811         struct bcm2835_host *host;
1812         unsigned long flags;
1813         struct mmc_request *mrq;
1814         struct dma_chan *terminate_chan = NULL;
1815
1816         host = (struct bcm2835_host *)param;
1817
1818         spin_lock_irqsave(&host->lock, flags);
1819
1820         log_event("TSK<", host->mrq, 0);
1821         /*
1822          * If this tasklet gets rescheduled while running, it will
1823          * be run again afterwards but without any active request.
1824          */
1825         if (!host->mrq) {
1826                 spin_unlock_irqrestore(&host->lock, flags);
1827                 return;
1828         }
1829
1830         del_timer(&host->timer);
1831
1832         mrq = host->mrq;
1833
1834         /* Drop the overclock after any data corruption, or after any
1835          * error while overclocked. Ignore errors for status commands,
1836          * as they are likely when a card is ejected. */
1837         if (host->overclock) {
1838                 if ((mrq->cmd && mrq->cmd->error &&
1839                      (mrq->cmd->opcode != MMC_SEND_STATUS)) ||
1840                     (mrq->data && mrq->data->error) ||
1841                     (mrq->stop && mrq->stop->error) ||
1842                     (mrq->sbc && mrq->sbc->error)) {
1843                         host->overclock_50--;
1844                         pr_warn("%s: reducing overclock due to errors\n",
1845                                 mmc_hostname(host->mmc));
1846                         host->reset_clock = 1;
1847                         mrq->cmd->error = -ETIMEDOUT;
1848                         mrq->cmd->retries = 1;
1849                 }
1850         }
1851
1852         host->mrq = NULL;
1853         host->cmd = NULL;
1854         host->data = NULL;
1855
1856         host->dma_desc = NULL;
1857         terminate_chan = host->dma_chan;
1858         host->dma_chan = NULL;
1859
1860         spin_unlock_irqrestore(&host->lock, flags);
1861
1862         if (terminate_chan)
1863         {
1864                 int err = dmaengine_terminate_all(terminate_chan);
1865                 if (err)
1866                         pr_err("%s: failed to terminate DMA (%d)\n",
1867                                mmc_hostname(host->mmc), err);
1868         }
1869
1870         /* The SDHOST block doesn't report any errors for a disconnected
1871            interface. All cards and SDIO devices should report some supported
1872            voltage range, so a zero response to SEND_OP_COND, IO_SEND_OP_COND
1873            or APP_SEND_OP_COND can be treated as an error. */
1874         if (((mrq->cmd->opcode == MMC_SEND_OP_COND) ||
1875              (mrq->cmd->opcode == SD_IO_SEND_OP_COND) ||
1876              (mrq->cmd->opcode == SD_APP_OP_COND)) &&
1877             (mrq->cmd->error == 0) &&
1878             (mrq->cmd->resp[0] == 0)) {
1879                 mrq->cmd->error = -ETIMEDOUT;
1880                 if (host->debug)
1881                         pr_info("%s: faking timeout due to zero OCR\n",
1882                                 mmc_hostname(host->mmc));
1883         }
1884
1885         mmc_request_done(host->mmc, mrq);
1886         log_event("TSK>", mrq, 0);
1887 }
1888
1889 int bcm2835_sdhost_add_host(struct bcm2835_host *host)
1890 {
1891         struct mmc_host *mmc;
1892         struct dma_slave_config cfg;
1893         char pio_limit_string[20];
1894         int ret;
1895
1896         mmc = host->mmc;
1897
1898         if (!mmc->f_max || mmc->f_max > host->max_clk)
1899                 mmc->f_max = host->max_clk;
1900         mmc->f_min = host->max_clk / SDCDIV_MAX_CDIV;
1901
1902         mmc->max_busy_timeout =  (~(unsigned int)0)/(mmc->f_max/1000);
1903
1904         pr_debug("f_max %d, f_min %d, max_busy_timeout %d\n",
1905                  mmc->f_max, mmc->f_min, mmc->max_busy_timeout);
1906
1907         /* host controller capabilities */
1908         mmc->caps |=
1909                 MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
1910                 MMC_CAP_NEEDS_POLL | MMC_CAP_HW_RESET |
1911                 ((ALLOW_CMD23_READ|ALLOW_CMD23_WRITE) * MMC_CAP_CMD23);
1912
1913         spin_lock_init(&host->lock);
1914
1915         if (host->allow_dma) {
1916                 if (IS_ERR_OR_NULL(host->dma_chan_rxtx)) {
1917                         pr_err("%s: unable to initialise DMA channel. "
1918                                "Falling back to PIO\n",
1919                                mmc_hostname(mmc));
1920                         host->use_dma = false;
1921                 } else {
1922                         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1923                         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1924                         cfg.slave_id = 13;              /* DREQ channel */
1925
1926                         /* Validate the slave configurations */
1927
1928                         cfg.direction = DMA_MEM_TO_DEV;
1929                         cfg.src_addr = 0;
1930                         cfg.dst_addr = host->bus_addr + SDDATA;
1931
1932                         ret = dmaengine_slave_config(host->dma_chan_rxtx, &cfg);
1933
1934                         if (ret == 0) {
1935                                 host->dma_cfg_tx = cfg;
1936
1937                                 cfg.direction = DMA_DEV_TO_MEM;
1938                                 cfg.src_addr = host->bus_addr + SDDATA;
1939                                 cfg.dst_addr = 0;
1940
1941                                 ret = dmaengine_slave_config(host->dma_chan_rxtx, &cfg);
1942                         }
1943
1944                         if (ret == 0) {
1945                                 host->dma_cfg_rx = cfg;
1946
1947                                 host->use_dma = true;
1948                         } else {
1949                                 pr_err("%s: unable to configure DMA channel. "
1950                                        "Falling back to PIO\n",
1951                                        mmc_hostname(mmc));
1952                                 dma_release_channel(host->dma_chan_rxtx);
1953                                 host->dma_chan_rxtx = NULL;
1954                                 host->use_dma = false;
1955                         }
1956                 }
1957         } else {
1958                 host->use_dma = false;
1959         }
1960
1961         mmc->max_segs = 128;
1962         mmc->max_req_size = 524288;
1963         mmc->max_seg_size = mmc->max_req_size;
1964         mmc->max_blk_size = 512;
1965         mmc->max_blk_count =  65535;
1966
1967         /* report supported voltage ranges */
1968         mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
1969
1970         tasklet_init(&host->finish_tasklet,
1971                 bcm2835_sdhost_tasklet_finish, (unsigned long)host);
1972
1973         INIT_WORK(&host->cmd_wait_wq, bcm2835_sdhost_cmd_wait_work);
1974
1975         timer_setup(&host->timer, bcm2835_sdhost_timeout, 0);
1976
1977         bcm2835_sdhost_init(host, 0);
1978
1979         ret = request_irq(host->irq, bcm2835_sdhost_irq, 0 /*IRQF_SHARED*/,
1980                                   mmc_hostname(mmc), host);
1981         if (ret) {
1982                 pr_err("%s: failed to request IRQ %d: %d\n",
1983                        mmc_hostname(mmc), host->irq, ret);
1984                 goto untasklet;
1985         }
1986
1987         mmc_add_host(mmc);
1988
1989         pio_limit_string[0] = '\0';
1990         if (host->use_dma && (host->pio_limit > 0))
1991                 sprintf(pio_limit_string, " (>%d)", host->pio_limit);
1992         pr_info("%s: %s loaded - DMA %s%s\n",
1993                 mmc_hostname(mmc), DRIVER_NAME,
1994                 host->use_dma ? "enabled" : "disabled",
1995                 pio_limit_string);
1996
1997         return 0;
1998
1999 untasklet:
2000         tasklet_kill(&host->finish_tasklet);
2001
2002         return ret;
2003 }
2004
2005 static int bcm2835_sdhost_probe(struct platform_device *pdev)
2006 {
2007         struct device *dev = &pdev->dev;
2008         struct device_node *node = dev->of_node;
2009         struct clk *clk;
2010         struct resource *iomem;
2011         struct bcm2835_host *host;
2012         struct mmc_host *mmc;
2013         const __be32 *addr;
2014         u32 msg[3];
2015         int na;
2016         int ret;
2017
2018         pr_debug("bcm2835_sdhost_probe\n");
2019         mmc = mmc_alloc_host(sizeof(*host), dev);
2020         if (!mmc)
2021                 return -ENOMEM;
2022
2023         mmc->ops = &bcm2835_sdhost_ops;
2024         host = mmc_priv(mmc);
2025         host->mmc = mmc;
2026         host->pio_timeout = msecs_to_jiffies(500);
2027         host->pio_limit = 1;
2028         host->max_delay = 1; /* Warn if over 1ms */
2029         host->allow_dma = 1;
2030         spin_lock_init(&host->lock);
2031
2032         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2033         host->ioaddr = devm_ioremap_resource(dev, iomem);
2034         if (IS_ERR(host->ioaddr)) {
2035                 ret = PTR_ERR(host->ioaddr);
2036                 goto err;
2037         }
2038
2039         na = of_n_addr_cells(node);
2040         addr = of_get_address(node, 0, NULL, NULL);
2041         if (!addr) {
2042                 dev_err(dev, "could not get DMA-register address\n");
2043                 return -ENODEV;
2044         }
2045         host->bus_addr = (phys_addr_t)of_read_number(addr, na);
2046         pr_debug(" - ioaddr %lx, iomem->start %lx, bus_addr %lx\n",
2047                  (unsigned long)host->ioaddr,
2048                  (unsigned long)iomem->start,
2049                  (unsigned long)host->bus_addr);
2050
2051         if (node) {
2052                 /* Read any custom properties */
2053                 of_property_read_u32(node,
2054                                      "brcm,delay-after-stop",
2055                                      &host->delay_after_stop);
2056                 of_property_read_u32(node,
2057                                      "brcm,overclock-50",
2058                                      &host->user_overclock_50);
2059                 of_property_read_u32(node,
2060                                      "brcm,pio-limit",
2061                                      &host->pio_limit);
2062                 host->allow_dma =
2063                         !of_property_read_bool(node, "brcm,force-pio");
2064                 host->debug = of_property_read_bool(node, "brcm,debug");
2065         }
2066
2067         host->dma_chan = NULL;
2068         host->dma_desc = NULL;
2069
2070         /* Formally recognise the other way of disabling DMA */
2071         if (host->pio_limit == 0x7fffffff)
2072                 host->allow_dma = false;
2073
2074         if (host->allow_dma) {
2075                 if (node) {
2076                         host->dma_chan_rxtx =
2077                                 dma_request_slave_channel(dev, "rx-tx");
2078                         if (!host->dma_chan_rxtx)
2079                                 host->dma_chan_rxtx =
2080                                         dma_request_slave_channel(dev, "tx");
2081                         if (!host->dma_chan_rxtx)
2082                                 host->dma_chan_rxtx =
2083                                         dma_request_slave_channel(dev, "rx");
2084                 } else {
2085                         dma_cap_mask_t mask;
2086
2087                         dma_cap_zero(mask);
2088                         /* we don't care about the channel, any would work */
2089                         dma_cap_set(DMA_SLAVE, mask);
2090                         host->dma_chan_rxtx =
2091                                 dma_request_channel(mask, NULL, NULL);
2092                 }
2093         }
2094
2095         clk = devm_clk_get(dev, NULL);
2096         if (IS_ERR(clk)) {
2097                 ret = PTR_ERR(clk);
2098                 if (ret == -EPROBE_DEFER)
2099                         dev_info(dev, "could not get clk, deferring probe\n");
2100                 else
2101                         dev_err(dev, "could not get clk\n");
2102                 goto err;
2103         }
2104
2105         host->fw = rpi_firmware_get(
2106                 of_parse_phandle(dev->of_node, "firmware", 0));
2107         if (!host->fw) {
2108                 ret = -EPROBE_DEFER;
2109                 goto err;
2110         }
2111
2112         host->max_clk = clk_get_rate(clk);
2113
2114         host->irq = platform_get_irq(pdev, 0);
2115         if (host->irq <= 0) {
2116                 dev_err(dev, "get IRQ failed\n");
2117                 ret = -EINVAL;
2118                 goto err;
2119         }
2120
2121         pr_debug(" - max_clk %lx, irq %d\n",
2122                  (unsigned long)host->max_clk,
2123                  (int)host->irq);
2124
2125         log_init(dev, iomem->start - host->bus_addr);
2126
2127         if (node)
2128                 mmc_of_parse(mmc);
2129         else
2130                 mmc->caps |= MMC_CAP_4_BIT_DATA;
2131
2132         msg[0] = 0;
2133         msg[1] = ~0;
2134         msg[2] = ~0;
2135
2136         rpi_firmware_property(host->fw,
2137                               RPI_FIRMWARE_SET_SDHOST_CLOCK,
2138                               &msg, sizeof(msg));
2139
2140         host->firmware_sets_cdiv = (msg[1] != ~0);
2141
2142         ret = bcm2835_sdhost_add_host(host);
2143         if (ret)
2144                 goto err;
2145
2146         platform_set_drvdata(pdev, host);
2147
2148         pr_debug("bcm2835_sdhost_probe -> OK\n");
2149
2150         return 0;
2151
2152 err:
2153         pr_debug("bcm2835_sdhost_probe -> err %d\n", ret);
2154         if (host->fw)
2155                 rpi_firmware_put(host->fw);
2156         if (host->dma_chan_rxtx)
2157                 dma_release_channel(host->dma_chan_rxtx);
2158         mmc_free_host(mmc);
2159
2160         return ret;
2161 }
2162
2163 static int bcm2835_sdhost_remove(struct platform_device *pdev)
2164 {
2165         struct bcm2835_host *host = platform_get_drvdata(pdev);
2166
2167         pr_debug("bcm2835_sdhost_remove\n");
2168
2169         mmc_remove_host(host->mmc);
2170
2171         bcm2835_sdhost_set_power(host, false);
2172
2173         free_irq(host->irq, host);
2174
2175         del_timer_sync(&host->timer);
2176
2177         tasklet_kill(&host->finish_tasklet);
2178         rpi_firmware_put(host->fw);
2179         if (host->dma_chan_rxtx)
2180                 dma_release_channel(host->dma_chan_rxtx);
2181         mmc_free_host(host->mmc);
2182         platform_set_drvdata(pdev, NULL);
2183
2184         pr_debug("bcm2835_sdhost_remove - OK\n");
2185         return 0;
2186 }
2187
2188 static const struct of_device_id bcm2835_sdhost_match[] = {
2189         { .compatible = "brcm,bcm2835-sdhost" },
2190         { }
2191 };
2192 MODULE_DEVICE_TABLE(of, bcm2835_sdhost_match);
2193
2194 static struct platform_driver bcm2835_sdhost_driver = {
2195         .probe      = bcm2835_sdhost_probe,
2196         .remove     = bcm2835_sdhost_remove,
2197         .driver     = {
2198                 .name           = DRIVER_NAME,
2199                 .owner          = THIS_MODULE,
2200                 .of_match_table = bcm2835_sdhost_match,
2201         },
2202 };
2203 module_platform_driver(bcm2835_sdhost_driver);
2204
2205 MODULE_ALIAS("platform:sdhost-bcm2835");
2206 MODULE_DESCRIPTION("BCM2835 SDHost driver");
2207 MODULE_LICENSE("GPL v2");
2208 MODULE_AUTHOR("Phil Elwell");