drivers/mmc/host/sdhci.c

   1 /*
   2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   3  *
   4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or (at
   9  * your option) any later version.
  10  *
  11  * Thanks to the following companies for their support:
  12  *
  13  *     - JMicron (hardware and technical support)
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/ktime.h>
  18 #include <linux/highmem.h>
  19 #include <linux/io.h>
  20 #include <linux/module.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sizes.h>
  25 #include <linux/swiotlb.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/of.h>
  29
  30 #include <linux/leds.h>
  31
  32 #include <linux/mmc/mmc.h>
  33 #include <linux/mmc/host.h>
  34 #include <linux/mmc/card.h>
  35 #include <linux/mmc/sdio.h>
  36 #include <linux/mmc/slot-gpio.h>
  37
  38 #include "sdhci.h"
  39
  40 #define DRIVER_NAME "sdhci"
  41
  42 #define DBG(f, x...) \
  43         pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45 #define SDHCI_DUMP(f, x...) \
  46         pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  47
  48 #define MAX_TUNING_LOOP 40
  49
  50 static unsigned int debug_quirks = 0;
  51 static unsigned int debug_quirks2;
  52
  53 static void sdhci_finish_data(struct sdhci_host *);
  54
  55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  56
  57 void sdhci_dumpregs(struct sdhci_host *host)
  58 {
  59         SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  60
  61         SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  62                    sdhci_readl(host, SDHCI_DMA_ADDRESS),
  63                    sdhci_readw(host, SDHCI_HOST_VERSION));
  64         SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  65                    sdhci_readw(host, SDHCI_BLOCK_SIZE),
  66                    sdhci_readw(host, SDHCI_BLOCK_COUNT));
  67         SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  68                    sdhci_readl(host, SDHCI_ARGUMENT),
  69                    sdhci_readw(host, SDHCI_TRANSFER_MODE));
  70         SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  71                    sdhci_readl(host, SDHCI_PRESENT_STATE),
  72                    sdhci_readb(host, SDHCI_HOST_CONTROL));
  73         SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  74                    sdhci_readb(host, SDHCI_POWER_CONTROL),
  75                    sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  76         SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  77                    sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  78                    sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  79         SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  80                    sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  81                    sdhci_readl(host, SDHCI_INT_STATUS));
  82         SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  83                    sdhci_readl(host, SDHCI_INT_ENABLE),
  84                    sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  85         SDHCI_DUMP("AC12 err:  0x%08x | Slot int: 0x%08x\n",
  86                    sdhci_readw(host, SDHCI_ACMD12_ERR),
  87                    sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  88         SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  89                    sdhci_readl(host, SDHCI_CAPABILITIES),
  90                    sdhci_readl(host, SDHCI_CAPABILITIES_1));
  91         SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  92                    sdhci_readw(host, SDHCI_COMMAND),
  93                    sdhci_readl(host, SDHCI_MAX_CURRENT));
  94         SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  95                    sdhci_readl(host, SDHCI_RESPONSE),
  96                    sdhci_readl(host, SDHCI_RESPONSE + 4));
  97         SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  98                    sdhci_readl(host, SDHCI_RESPONSE + 8),
  99                    sdhci_readl(host, SDHCI_RESPONSE + 12));
 100         SDHCI_DUMP("Host ctl2: 0x%08x\n",
 101                    sdhci_readw(host, SDHCI_HOST_CONTROL2));
 102
 103         if (host->flags & SDHCI_USE_ADMA) {
 104                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
 105                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 106                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 107                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 108                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 109                 } else {
 110                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 111                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 112                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 113                 }
 114         }
 115
 116         SDHCI_DUMP("============================================\n");
 117 }
 118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120 /*****************************************************************************\
 121  *                                                                           *
 122  * Low level functions                                                       *
 123  *                                                                           *
 124 \*****************************************************************************/
 125
 126 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 127 {
 128         return cmd->data || cmd->flags & MMC_RSP_BUSY;
 129 }
 130
 131 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 132 {
 133         u32 present;
 134
 135         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 136             !mmc_card_is_removable(host->mmc))
 137                 return;
 138
 139         if (enable) {
 140                 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 141                                       SDHCI_CARD_PRESENT;
 142
 143                 host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 144                                        SDHCI_INT_CARD_INSERT;
 145         } else {
 146                 host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 147         }
 148
 149         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 150         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 151 }
 152
 153 static void sdhci_enable_card_detection(struct sdhci_host *host)
 154 {
 155         sdhci_set_card_detection(host, true);
 156 }
 157
 158 static void sdhci_disable_card_detection(struct sdhci_host *host)
 159 {
 160         sdhci_set_card_detection(host, false);
 161 }
 162
 163 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 164 {
 165         if (host->bus_on)
 166                 return;
 167         host->bus_on = true;
 168         pm_runtime_get_noresume(host->mmc->parent);
 169 }
 170
 171 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 172 {
 173         if (!host->bus_on)
 174                 return;
 175         host->bus_on = false;
 176         pm_runtime_put_noidle(host->mmc->parent);
 177 }
 178
 179 void sdhci_reset(struct sdhci_host *host, u8 mask)
 180 {
 181         ktime_t timeout;
 182
 183         sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 184
 185         if (mask & SDHCI_RESET_ALL) {
 186                 host->clock = 0;
 187                 /* Reset-all turns off SD Bus Power */
 188                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 189                         sdhci_runtime_pm_bus_off(host);
 190         }
 191
 192         /* Wait max 100 ms */
 193         timeout = ktime_add_ms(ktime_get(), 100);
 194
 195         /* hw clears the bit when it's done */
 196         while (sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask) {
 197                 if (ktime_after(ktime_get(), timeout)) {
 198                         pr_err("%s: Reset 0x%x never completed.\n",
 199                                 mmc_hostname(host->mmc), (int)mask);
 200                         sdhci_dumpregs(host);
 201                         return;
 202                 }
 203                 udelay(10);
 204         }
 205 }
 206 EXPORT_SYMBOL_GPL(sdhci_reset);
 207
 208 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 209 {
 210         if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 211                 struct mmc_host *mmc = host->mmc;
 212
 213                 if (!mmc->ops->get_cd(mmc))
 214                         return;
 215         }
 216
 217         host->ops->reset(host, mask);
 218
 219         if (mask & SDHCI_RESET_ALL) {
 220                 if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 221                         if (host->ops->enable_dma)
 222                                 host->ops->enable_dma(host);
 223                 }
 224
 225                 /* Resetting the controller clears many */
 226                 host->preset_enabled = false;
 227         }
 228 }
 229
 230 static void sdhci_set_default_irqs(struct sdhci_host *host)
 231 {
 232         host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 233                     SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 234                     SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 235                     SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 236                     SDHCI_INT_RESPONSE;
 237
 238         if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 239             host->tuning_mode == SDHCI_TUNING_MODE_3)
 240                 host->ier |= SDHCI_INT_RETUNE;
 241
 242         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 243         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 244 }
 245
 246 static void sdhci_init(struct sdhci_host *host, int soft)
 247 {
 248         struct mmc_host *mmc = host->mmc;
 249
 250         if (soft)
 251                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 252         else
 253                 sdhci_do_reset(host, SDHCI_RESET_ALL);
 254
 255         sdhci_set_default_irqs(host);
 256
 257         host->cqe_on = false;
 258
 259         if (soft) {
 260                 /* force clock reconfiguration */
 261                 host->clock = 0;
 262                 mmc->ops->set_ios(mmc, &mmc->ios);
 263         }
 264 }
 265
 266 static void sdhci_reinit(struct sdhci_host *host)
 267 {
 268         sdhci_init(host, 0);
 269         sdhci_enable_card_detection(host);
 270 }
 271
 272 static void __sdhci_led_activate(struct sdhci_host *host)
 273 {
 274         u8 ctrl;
 275
 276         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 277         ctrl |= SDHCI_CTRL_LED;
 278         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 279 }
 280
 281 static void __sdhci_led_deactivate(struct sdhci_host *host)
 282 {
 283         u8 ctrl;
 284
 285         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 286         ctrl &= ~SDHCI_CTRL_LED;
 287         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 288 }
 289
 290 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
 291 static void sdhci_led_control(struct led_classdev *led,
 292                               enum led_brightness brightness)
 293 {
 294         struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 295         unsigned long flags;
 296
 297         spin_lock_irqsave(&host->lock, flags);
 298
 299         if (host->runtime_suspended)
 300                 goto out;
 301
 302         if (brightness == LED_OFF)
 303                 __sdhci_led_deactivate(host);
 304         else
 305                 __sdhci_led_activate(host);
 306 out:
 307         spin_unlock_irqrestore(&host->lock, flags);
 308 }
 309
 310 static int sdhci_led_register(struct sdhci_host *host)
 311 {
 312         struct mmc_host *mmc = host->mmc;
 313
 314         snprintf(host->led_name, sizeof(host->led_name),
 315                  "%s::", mmc_hostname(mmc));
 316
 317         host->led.name = host->led_name;
 318         host->led.brightness = LED_OFF;
 319         host->led.default_trigger = mmc_hostname(mmc);
 320         host->led.brightness_set = sdhci_led_control;
 321
 322         return led_classdev_register(mmc_dev(mmc), &host->led);
 323 }
 324
 325 static void sdhci_led_unregister(struct sdhci_host *host)
 326 {
 327         led_classdev_unregister(&host->led);
 328 }
 329
 330 static inline void sdhci_led_activate(struct sdhci_host *host)
 331 {
 332 }
 333
 334 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 335 {
 336 }
 337
 338 #else
 339
 340 static inline int sdhci_led_register(struct sdhci_host *host)
 341 {
 342         return 0;
 343 }
 344
 345 static inline void sdhci_led_unregister(struct sdhci_host *host)
 346 {
 347 }
 348
 349 static inline void sdhci_led_activate(struct sdhci_host *host)
 350 {
 351         __sdhci_led_activate(host);
 352 }
 353
 354 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 355 {
 356         __sdhci_led_deactivate(host);
 357 }
 358
 359 #endif
 360
 361 /*****************************************************************************\
 362  *                                                                           *
 363  * Core functions                                                            *
 364  *                                                                           *
 365 \*****************************************************************************/
 366
 367 static void sdhci_read_block_pio(struct sdhci_host *host)
 368 {
 369         unsigned long flags;
 370         size_t blksize, len, chunk;
 371         u32 uninitialized_var(scratch);
 372         u8 *buf;
 373
 374         DBG("PIO reading\n");
 375
 376         blksize = host->data->blksz;
 377         chunk = 0;
 378
 379         local_irq_save(flags);
 380
 381         while (blksize) {
 382                 BUG_ON(!sg_miter_next(&host->sg_miter));
 383
 384                 len = min(host->sg_miter.length, blksize);
 385
 386                 blksize -= len;
 387                 host->sg_miter.consumed = len;
 388
 389                 buf = host->sg_miter.addr;
 390
 391                 while (len) {
 392                         if (chunk == 0) {
 393                                 scratch = sdhci_readl(host, SDHCI_BUFFER);
 394                                 chunk = 4;
 395                         }
 396
 397                         *buf = scratch & 0xFF;
 398
 399                         buf++;
 400                         scratch >>= 8;
 401                         chunk--;
 402                         len--;
 403                 }
 404         }
 405
 406         sg_miter_stop(&host->sg_miter);
 407
 408         local_irq_restore(flags);
 409 }
 410
 411 static void sdhci_write_block_pio(struct sdhci_host *host)
 412 {
 413         unsigned long flags;
 414         size_t blksize, len, chunk;
 415         u32 scratch;
 416         u8 *buf;
 417
 418         DBG("PIO writing\n");
 419
 420         blksize = host->data->blksz;
 421         chunk = 0;
 422         scratch = 0;
 423
 424         local_irq_save(flags);
 425
 426         while (blksize) {
 427                 BUG_ON(!sg_miter_next(&host->sg_miter));
 428
 429                 len = min(host->sg_miter.length, blksize);
 430
 431                 blksize -= len;
 432                 host->sg_miter.consumed = len;
 433
 434                 buf = host->sg_miter.addr;
 435
 436                 while (len) {
 437                         scratch |= (u32)*buf << (chunk * 8);
 438
 439                         buf++;
 440                         chunk++;
 441                         len--;
 442
 443                         if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 444                                 sdhci_writel(host, scratch, SDHCI_BUFFER);
 445                                 chunk = 0;
 446                                 scratch = 0;
 447                         }
 448                 }
 449         }
 450
 451         sg_miter_stop(&host->sg_miter);
 452
 453         local_irq_restore(flags);
 454 }
 455
 456 static void sdhci_transfer_pio(struct sdhci_host *host)
 457 {
 458         u32 mask;
 459
 460         if (host->blocks == 0)
 461                 return;
 462
 463         if (host->data->flags & MMC_DATA_READ)
 464                 mask = SDHCI_DATA_AVAILABLE;
 465         else
 466                 mask = SDHCI_SPACE_AVAILABLE;
 467
 468         /*
 469          * Some controllers (JMicron JMB38x) mess up the buffer bits
 470          * for transfers < 4 bytes. As long as it is just one block,
 471          * we can ignore the bits.
 472          */
 473         if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 474                 (host->data->blocks == 1))
 475                 mask = ~0;
 476
 477         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 478                 if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 479                         udelay(100);
 480
 481                 if (host->data->flags & MMC_DATA_READ)
 482                         sdhci_read_block_pio(host);
 483                 else
 484                         sdhci_write_block_pio(host);
 485
 486                 host->blocks--;
 487                 if (host->blocks == 0)
 488                         break;
 489         }
 490
 491         DBG("PIO transfer complete.\n");
 492 }
 493
 494 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 495                                   struct mmc_data *data, int cookie)
 496 {
 497         int sg_count;
 498
 499         /*
 500          * If the data buffers are already mapped, return the previous
 501          * dma_map_sg() result.
 502          */
 503         if (data->host_cookie == COOKIE_PRE_MAPPED)
 504                 return data->sg_count;
 505
 506         /* Bounce write requests to the bounce buffer */
 507         if (host->bounce_buffer) {
 508                 unsigned int length = data->blksz * data->blocks;
 509
 510                 if (length > host->bounce_buffer_size) {
 511                         pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 512                                mmc_hostname(host->mmc), length,
 513                                host->bounce_buffer_size);
 514                         return -EIO;
 515                 }
 516                 if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 517                         /* Copy the data to the bounce buffer */
 518                         sg_copy_to_buffer(data->sg, data->sg_len,
 519                                           host->bounce_buffer,
 520                                           length);
 521                 }
 522                 /* Switch ownership to the DMA */
 523                 dma_sync_single_for_device(host->mmc->parent,
 524                                            host->bounce_addr,
 525                                            host->bounce_buffer_size,
 526                                            mmc_get_dma_dir(data));
 527                 /* Just a dummy value */
 528                 sg_count = 1;
 529         } else {
 530                 /* Just access the data directly from memory */
 531                 sg_count = dma_map_sg(mmc_dev(host->mmc),
 532                                       data->sg, data->sg_len,
 533                                       mmc_get_dma_dir(data));
 534         }
 535
 536         if (sg_count == 0)
 537                 return -ENOSPC;
 538
 539         data->sg_count = sg_count;
 540         data->host_cookie = cookie;
 541
 542         return sg_count;
 543 }
 544
 545 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 546 {
 547         local_irq_save(*flags);
 548         return kmap_atomic(sg_page(sg)) + sg->offset;
 549 }
 550
 551 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 552 {
 553         kunmap_atomic(buffer);
 554         local_irq_restore(*flags);
 555 }
 556
 557 static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
 558                                   dma_addr_t addr, int len, unsigned cmd)
 559 {
 560         struct sdhci_adma2_64_desc *dma_desc = desc;
 561
 562         /* 32-bit and 64-bit descriptors have these members in same position */
 563         dma_desc->cmd = cpu_to_le16(cmd);
 564         dma_desc->len = cpu_to_le16(len);
 565         dma_desc->addr_lo = cpu_to_le32((u32)addr);
 566
 567         if (host->flags & SDHCI_USE_64_BIT_DMA)
 568                 dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
 569 }
 570
 571 static void sdhci_adma_mark_end(void *desc)
 572 {
 573         struct sdhci_adma2_64_desc *dma_desc = desc;
 574
 575         /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 576         dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 577 }
 578
 579 static void sdhci_adma_table_pre(struct sdhci_host *host,
 580         struct mmc_data *data, int sg_count)
 581 {
 582         struct scatterlist *sg;
 583         unsigned long flags;
 584         dma_addr_t addr, align_addr;
 585         void *desc, *align;
 586         char *buffer;
 587         int len, offset, i;
 588
 589         /*
 590          * The spec does not specify endianness of descriptor table.
 591          * We currently guess that it is LE.
 592          */
 593
 594         host->sg_count = sg_count;
 595
 596         desc = host->adma_table;
 597         align = host->align_buffer;
 598
 599         align_addr = host->align_addr;
 600
 601         for_each_sg(data->sg, sg, host->sg_count, i) {
 602                 addr = sg_dma_address(sg);
 603                 len = sg_dma_len(sg);
 604
 605                 /*
 606                  * The SDHCI specification states that ADMA addresses must
 607                  * be 32-bit aligned. If they aren't, then we use a bounce
 608                  * buffer for the (up to three) bytes that screw up the
 609                  * alignment.
 610                  */
 611                 offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 612                          SDHCI_ADMA2_MASK;
 613                 if (offset) {
 614                         if (data->flags & MMC_DATA_WRITE) {
 615                                 buffer = sdhci_kmap_atomic(sg, &flags);
 616                                 memcpy(align, buffer, offset);
 617                                 sdhci_kunmap_atomic(buffer, &flags);
 618                         }
 619
 620                         /* tran, valid */
 621                         sdhci_adma_write_desc(host, desc, align_addr, offset,
 622                                               ADMA2_TRAN_VALID);
 623
 624                         BUG_ON(offset > 65536);
 625
 626                         align += SDHCI_ADMA2_ALIGN;
 627                         align_addr += SDHCI_ADMA2_ALIGN;
 628
 629                         desc += host->desc_sz;
 630
 631                         addr += offset;
 632                         len -= offset;
 633                 }
 634
 635                 BUG_ON(len > 65536);
 636
 637                 if (len) {
 638                         /* tran, valid */
 639                         sdhci_adma_write_desc(host, desc, addr, len,
 640                                               ADMA2_TRAN_VALID);
 641                         desc += host->desc_sz;
 642                 }
 643
 644                 /*
 645                  * If this triggers then we have a calculation bug
 646                  * somewhere. :/
 647                  */
 648                 WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 649         }
 650
 651         if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 652                 /* Mark the last descriptor as the terminating descriptor */
 653                 if (desc != host->adma_table) {
 654                         desc -= host->desc_sz;
 655                         sdhci_adma_mark_end(desc);
 656                 }
 657         } else {
 658                 /* Add a terminating entry - nop, end, valid */
 659                 sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
 660         }
 661 }
 662
 663 static void sdhci_adma_table_post(struct sdhci_host *host,
 664         struct mmc_data *data)
 665 {
 666         struct scatterlist *sg;
 667         int i, size;
 668         void *align;
 669         char *buffer;
 670         unsigned long flags;
 671
 672         if (data->flags & MMC_DATA_READ) {
 673                 bool has_unaligned = false;
 674
 675                 /* Do a quick scan of the SG list for any unaligned mappings */
 676                 for_each_sg(data->sg, sg, host->sg_count, i)
 677                         if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 678                                 has_unaligned = true;
 679                                 break;
 680                         }
 681
 682                 if (has_unaligned) {
 683                         dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 684                                             data->sg_len, DMA_FROM_DEVICE);
 685
 686                         align = host->align_buffer;
 687
 688                         for_each_sg(data->sg, sg, host->sg_count, i) {
 689                                 if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 690                                         size = SDHCI_ADMA2_ALIGN -
 691                                                (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 692
 693                                         buffer = sdhci_kmap_atomic(sg, &flags);
 694                                         memcpy(buffer, align, size);
 695                                         sdhci_kunmap_atomic(buffer, &flags);
 696
 697                                         align += SDHCI_ADMA2_ALIGN;
 698                                 }
 699                         }
 700                 }
 701         }
 702 }
 703
 704 static u32 sdhci_sdma_address(struct sdhci_host *host)
 705 {
 706         if (host->bounce_buffer)
 707                 return host->bounce_addr;
 708         else
 709                 return sg_dma_address(host->data->sg);
 710 }
 711
 712 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 713 {
 714         u8 count;
 715         struct mmc_data *data = cmd->data;
 716         unsigned target_timeout, current_timeout;
 717
 718         /*
 719          * If the host controller provides us with an incorrect timeout
 720          * value, just skip the check and use 0xE.  The hardware may take
 721          * longer to time out, but that's much better than having a too-short
 722          * timeout value.
 723          */
 724         if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 725                 return 0xE;
 726
 727         /* Unspecified timeout, assume max */
 728         if (!data && !cmd->busy_timeout)
 729                 return 0xE;
 730
 731         /* timeout in us */
 732         if (!data)
 733                 target_timeout = cmd->busy_timeout * 1000;
 734         else {
 735                 target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 736                 if (host->clock && data->timeout_clks) {
 737                         unsigned long long val;
 738
 739                         /*
 740                          * data->timeout_clks is in units of clock cycles.
 741                          * host->clock is in Hz.  target_timeout is in us.
 742                          * Hence, us = 1000000 * cycles / Hz.  Round up.
 743                          */
 744                         val = 1000000ULL * data->timeout_clks;
 745                         if (do_div(val, host->clock))
 746                                 target_timeout++;
 747                         target_timeout += val;
 748                 }
 749         }
 750
 751         /*
 752          * Figure out needed cycles.
 753          * We do this in steps in order to fit inside a 32 bit int.
 754          * The first step is the minimum timeout, which will have a
 755          * minimum resolution of 6 bits:
 756          * (1) 2^13*1000 > 2^22,
 757          * (2) host->timeout_clk < 2^16
 758          *     =>
 759          *     (1) / (2) > 2^6
 760          */
 761         count = 0;
 762         current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 763         while (current_timeout < target_timeout) {
 764                 count++;
 765                 current_timeout <<= 1;
 766                 if (count >= 0xF)
 767                         break;
 768         }
 769
 770         if (count >= 0xF) {
 771                 DBG("Too large timeout 0x%x requested for CMD%d!\n",
 772                     count, cmd->opcode);
 773                 count = 0xE;
 774         }
 775
 776         return count;
 777 }
 778
 779 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 780 {
 781         u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 782         u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 783
 784         if (host->flags & SDHCI_REQ_USE_DMA)
 785                 host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 786         else
 787                 host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 788
 789         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 790         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 791 }
 792
 793 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 794 {
 795         u8 count;
 796
 797         if (host->ops->set_timeout) {
 798                 host->ops->set_timeout(host, cmd);
 799         } else {
 800                 count = sdhci_calc_timeout(host, cmd);
 801                 sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 802         }
 803 }
 804
 805 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 806 {
 807         u8 ctrl;
 808         struct mmc_data *data = cmd->data;
 809
 810         if (sdhci_data_line_cmd(cmd))
 811                 sdhci_set_timeout(host, cmd);
 812
 813         if (!data)
 814                 return;
 815
 816         WARN_ON(host->data);
 817
 818         /* Sanity checks */
 819         BUG_ON(data->blksz * data->blocks > 524288);
 820         BUG_ON(data->blksz > host->mmc->max_blk_size);
 821         BUG_ON(data->blocks > 65535);
 822
 823         host->data = data;
 824         host->data_early = 0;
 825         host->data->bytes_xfered = 0;
 826
 827         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 828                 struct scatterlist *sg;
 829                 unsigned int length_mask, offset_mask;
 830                 int i;
 831
 832                 host->flags |= SDHCI_REQ_USE_DMA;
 833
 834                 /*
 835                  * FIXME: This doesn't account for merging when mapping the
 836                  * scatterlist.
 837                  *
 838                  * The assumption here being that alignment and lengths are
 839                  * the same after DMA mapping to device address space.
 840                  */
 841                 length_mask = 0;
 842                 offset_mask = 0;
 843                 if (host->flags & SDHCI_USE_ADMA) {
 844                         if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
 845                                 length_mask = 3;
 846                                 /*
 847                                  * As we use up to 3 byte chunks to work
 848                                  * around alignment problems, we need to
 849                                  * check the offset as well.
 850                                  */
 851                                 offset_mask = 3;
 852                         }
 853                 } else {
 854                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
 855                                 length_mask = 3;
 856                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
 857                                 offset_mask = 3;
 858                 }
 859
 860                 if (unlikely(length_mask | offset_mask)) {
 861                         for_each_sg(data->sg, sg, data->sg_len, i) {
 862                                 if (sg->length & length_mask) {
 863                                         DBG("Reverting to PIO because of transfer size (%d)\n",
 864                                             sg->length);
 865                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 866                                         break;
 867                                 }
 868                                 if (sg->offset & offset_mask) {
 869                                         DBG("Reverting to PIO because of bad alignment\n");
 870                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 871                                         break;
 872                                 }
 873                         }
 874                 }
 875         }
 876
 877         if (host->flags & SDHCI_REQ_USE_DMA) {
 878                 int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
 879
 880                 if (sg_cnt <= 0) {
 881                         /*
 882                          * This only happens when someone fed
 883                          * us an invalid request.
 884                          */
 885                         WARN_ON(1);
 886                         host->flags &= ~SDHCI_REQ_USE_DMA;
 887                 } else if (host->flags & SDHCI_USE_ADMA) {
 888                         sdhci_adma_table_pre(host, data, sg_cnt);
 889
 890                         sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
 891                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 892                                 sdhci_writel(host,
 893                                              (u64)host->adma_addr >> 32,
 894                                              SDHCI_ADMA_ADDRESS_HI);
 895                 } else {
 896                         WARN_ON(sg_cnt != 1);
 897                         sdhci_writel(host, sdhci_sdma_address(host),
 898                                      SDHCI_DMA_ADDRESS);
 899                 }
 900         }
 901
 902         /*
 903          * Always adjust the DMA selection as some controllers
 904          * (e.g. JMicron) can't do PIO properly when the selection
 905          * is ADMA.
 906          */
 907         if (host->version >= SDHCI_SPEC_200) {
 908                 ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 909                 ctrl &= ~SDHCI_CTRL_DMA_MASK;
 910                 if ((host->flags & SDHCI_REQ_USE_DMA) &&
 911                         (host->flags & SDHCI_USE_ADMA)) {
 912                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 913                                 ctrl |= SDHCI_CTRL_ADMA64;
 914                         else
 915                                 ctrl |= SDHCI_CTRL_ADMA32;
 916                 } else {
 917                         ctrl |= SDHCI_CTRL_SDMA;
 918                 }
 919                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 920         }
 921
 922         if (!(host->flags & SDHCI_REQ_USE_DMA)) {
 923                 int flags;
 924
 925                 flags = SG_MITER_ATOMIC;
 926                 if (host->data->flags & MMC_DATA_READ)
 927                         flags |= SG_MITER_TO_SG;
 928                 else
 929                         flags |= SG_MITER_FROM_SG;
 930                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 931                 host->blocks = data->blocks;
 932         }
 933
 934         sdhci_set_transfer_irqs(host);
 935
 936         /* Set the DMA boundary value and block size */
 937         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
 938                      SDHCI_BLOCK_SIZE);
 939         sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
 940 }
 941
 942 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
 943                                     struct mmc_request *mrq)
 944 {
 945         return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
 946                !mrq->cap_cmd_during_tfr;
 947 }
 948
 949 static void sdhci_set_transfer_mode(struct sdhci_host *host,
 950         struct mmc_command *cmd)
 951 {
 952         u16 mode = 0;
 953         struct mmc_data *data = cmd->data;
 954
 955         if (data == NULL) {
 956                 if (host->quirks2 &
 957                         SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
 958                         sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
 959                 } else {
 960                 /* clear Auto CMD settings for no data CMDs */
 961                         mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
 962                         sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
 963                                 SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
 964                 }
 965                 return;
 966         }
 967
 968         WARN_ON(!host->data);
 969
 970         if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
 971                 mode = SDHCI_TRNS_BLK_CNT_EN;
 972
 973         if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
 974                 mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
 975                 /*
 976                  * If we are sending CMD23, CMD12 never gets sent
 977                  * on successful completion (so no Auto-CMD12).
 978                  */
 979                 if (sdhci_auto_cmd12(host, cmd->mrq) &&
 980                     (cmd->opcode != SD_IO_RW_EXTENDED))
 981                         mode |= SDHCI_TRNS_AUTO_CMD12;
 982                 else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
 983                         mode |= SDHCI_TRNS_AUTO_CMD23;
 984                         sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
 985                 }
 986         }
 987
 988         if (data->flags & MMC_DATA_READ)
 989                 mode |= SDHCI_TRNS_READ;
 990         if (host->flags & SDHCI_REQ_USE_DMA)
 991                 mode |= SDHCI_TRNS_DMA;
 992
 993         sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
 994 }
 995
 996 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
 997 {
 998         return (!(host->flags & SDHCI_DEVICE_DEAD) &&
 999                 ((mrq->cmd && mrq->cmd->error) ||
1000                  (mrq->sbc && mrq->sbc->error) ||
1001                  (mrq->data && ((mrq->data->error && !mrq->data->stop) ||
1002                                 (mrq->data->stop && mrq->data->stop->error))) ||
1003                  (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1004 }
1005
1006 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1007 {
1008         int i;
1009
1010         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1011                 if (host->mrqs_done[i] == mrq) {
1012                         WARN_ON(1);
1013                         return;
1014                 }
1015         }
1016
1017         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1018                 if (!host->mrqs_done[i]) {
1019                         host->mrqs_done[i] = mrq;
1020                         break;
1021                 }
1022         }
1023
1024         WARN_ON(i >= SDHCI_MAX_MRQS);
1025
1026         tasklet_schedule(&host->finish_tasklet);
1027 }
1028
1029 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1030 {
1031         if (host->cmd && host->cmd->mrq == mrq)
1032                 host->cmd = NULL;
1033
1034         if (host->data_cmd && host->data_cmd->mrq == mrq)
1035                 host->data_cmd = NULL;
1036
1037         if (host->data && host->data->mrq == mrq)
1038                 host->data = NULL;
1039
1040         if (sdhci_needs_reset(host, mrq))
1041                 host->pending_reset = true;
1042
1043         __sdhci_finish_mrq(host, mrq);
1044 }
1045
1046 static void sdhci_finish_data(struct sdhci_host *host)
1047 {
1048         struct mmc_command *data_cmd = host->data_cmd;
1049         struct mmc_data *data = host->data;
1050
1051         host->data = NULL;
1052         host->data_cmd = NULL;
1053
1054         if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1055             (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1056                 sdhci_adma_table_post(host, data);
1057
1058         /*
1059          * The specification states that the block count register must
1060          * be updated, but it does not specify at what point in the
1061          * data flow. That makes the register entirely useless to read
1062          * back so we have to assume that nothing made it to the card
1063          * in the event of an error.
1064          */
1065         if (data->error)
1066                 data->bytes_xfered = 0;
1067         else
1068                 data->bytes_xfered = data->blksz * data->blocks;
1069
1070         /*
1071          * Need to send CMD12 if -
1072          * a) open-ended multiblock transfer (no CMD23)
1073          * b) error in multiblock transfer
1074          */
1075         if (data->stop &&
1076             (data->error ||
1077              !data->mrq->sbc)) {
1078
1079                 /*
1080                  * The controller needs a reset of internal state machines
1081                  * upon error conditions.
1082                  */
1083                 if (data->error) {
1084                         if (!host->cmd || host->cmd == data_cmd)
1085                                 sdhci_do_reset(host, SDHCI_RESET_CMD);
1086                         sdhci_do_reset(host, SDHCI_RESET_DATA);
1087                 }
1088
1089                 /*
1090                  * 'cap_cmd_during_tfr' request must not use the command line
1091                  * after mmc_command_done() has been called. It is upper layer's
1092                  * responsibility to send the stop command if required.
1093                  */
1094                 if (data->mrq->cap_cmd_during_tfr) {
1095                         sdhci_finish_mrq(host, data->mrq);
1096                 } else {
1097                         /* Avoid triggering warning in sdhci_send_command() */
1098                         host->cmd = NULL;
1099                         sdhci_send_command(host, data->stop);
1100                 }
1101         } else {
1102                 sdhci_finish_mrq(host, data->mrq);
1103         }
1104 }
1105
1106 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1107                             unsigned long timeout)
1108 {
1109         if (sdhci_data_line_cmd(mrq->cmd))
1110                 mod_timer(&host->data_timer, timeout);
1111         else
1112                 mod_timer(&host->timer, timeout);
1113 }
1114
1115 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1116 {
1117         if (sdhci_data_line_cmd(mrq->cmd))
1118                 del_timer(&host->data_timer);
1119         else
1120                 del_timer(&host->timer);
1121 }
1122
1123 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1124 {
1125         int flags;
1126         u32 mask;
1127         unsigned long timeout;
1128
1129         WARN_ON(host->cmd);
1130
1131         /* Initially, a command has no error */
1132         cmd->error = 0;
1133
1134         if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1135             cmd->opcode == MMC_STOP_TRANSMISSION)
1136                 cmd->flags |= MMC_RSP_BUSY;
1137
1138         /* Wait max 10 ms */
1139         timeout = 10;
1140
1141         mask = SDHCI_CMD_INHIBIT;
1142         if (sdhci_data_line_cmd(cmd))
1143                 mask |= SDHCI_DATA_INHIBIT;
1144
1145         /* We shouldn't wait for data inihibit for stop commands, even
1146            though they might use busy signaling */
1147         if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1148                 mask &= ~SDHCI_DATA_INHIBIT;
1149
1150         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1151                 if (timeout == 0) {
1152                         pr_err("%s: Controller never released inhibit bit(s).\n",
1153                                mmc_hostname(host->mmc));
1154                         sdhci_dumpregs(host);
1155                         cmd->error = -EIO;
1156                         sdhci_finish_mrq(host, cmd->mrq);
1157                         return;
1158                 }
1159                 timeout--;
1160                 mdelay(1);
1161         }
1162
1163         timeout = jiffies;
1164         if (!cmd->data && cmd->busy_timeout > 9000)
1165                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1166         else
1167                 timeout += 10 * HZ;
1168         sdhci_mod_timer(host, cmd->mrq, timeout);
1169
1170         host->cmd = cmd;
1171         if (sdhci_data_line_cmd(cmd)) {
1172                 WARN_ON(host->data_cmd);
1173                 host->data_cmd = cmd;
1174         }
1175
1176         sdhci_prepare_data(host, cmd);
1177
1178         sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1179
1180         sdhci_set_transfer_mode(host, cmd);
1181
1182         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1183                 pr_err("%s: Unsupported response type!\n",
1184                         mmc_hostname(host->mmc));
1185                 cmd->error = -EINVAL;
1186                 sdhci_finish_mrq(host, cmd->mrq);
1187                 return;
1188         }
1189
1190         if (!(cmd->flags & MMC_RSP_PRESENT))
1191                 flags = SDHCI_CMD_RESP_NONE;
1192         else if (cmd->flags & MMC_RSP_136)
1193                 flags = SDHCI_CMD_RESP_LONG;
1194         else if (cmd->flags & MMC_RSP_BUSY)
1195                 flags = SDHCI_CMD_RESP_SHORT_BUSY;
1196         else
1197                 flags = SDHCI_CMD_RESP_SHORT;
1198
1199         if (cmd->flags & MMC_RSP_CRC)
1200                 flags |= SDHCI_CMD_CRC;
1201         if (cmd->flags & MMC_RSP_OPCODE)
1202                 flags |= SDHCI_CMD_INDEX;
1203
1204         /* CMD19 is special in that the Data Present Select should be set */
1205         if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1206             cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1207                 flags |= SDHCI_CMD_DATA;
1208
1209         sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1210 }
1211 EXPORT_SYMBOL_GPL(sdhci_send_command);
1212
1213 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1214 {
1215         int i, reg;
1216
1217         for (i = 0; i < 4; i++) {
1218                 reg = SDHCI_RESPONSE + (3 - i) * 4;
1219                 cmd->resp[i] = sdhci_readl(host, reg);
1220         }
1221
1222         if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1223                 return;
1224
1225         /* CRC is stripped so we need to do some shifting */
1226         for (i = 0; i < 4; i++) {
1227                 cmd->resp[i] <<= 8;
1228                 if (i != 3)
1229                         cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1230         }
1231 }
1232
1233 static void sdhci_finish_command(struct sdhci_host *host)
1234 {
1235         struct mmc_command *cmd = host->cmd;
1236
1237         host->cmd = NULL;
1238
1239         if (cmd->flags & MMC_RSP_PRESENT) {
1240                 if (cmd->flags & MMC_RSP_136) {
1241                         sdhci_read_rsp_136(host, cmd);
1242                 } else {
1243                         cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1244                 }
1245         }
1246
1247         if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1248                 mmc_command_done(host->mmc, cmd->mrq);
1249
1250         /*
1251          * The host can send and interrupt when the busy state has
1252          * ended, allowing us to wait without wasting CPU cycles.
1253          * The busy signal uses DAT0 so this is similar to waiting
1254          * for data to complete.
1255          *
1256          * Note: The 1.0 specification is a bit ambiguous about this
1257          *       feature so there might be some problems with older
1258          *       controllers.
1259          */
1260         if (cmd->flags & MMC_RSP_BUSY) {
1261                 if (cmd->data) {
1262                         DBG("Cannot wait for busy signal when also doing a data transfer");
1263                 } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1264                            cmd == host->data_cmd) {
1265                         /* Command complete before busy is ended */
1266                         return;
1267                 }
1268         }
1269
1270         /* Finished CMD23, now send actual command. */
1271         if (cmd == cmd->mrq->sbc) {
1272                 sdhci_send_command(host, cmd->mrq->cmd);
1273         } else {
1274
1275                 /* Processed actual command. */
1276                 if (host->data && host->data_early)
1277                         sdhci_finish_data(host);
1278
1279                 if (!cmd->data)
1280                         sdhci_finish_mrq(host, cmd->mrq);
1281         }
1282 }
1283
1284 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1285 {
1286         u16 preset = 0;
1287
1288         switch (host->timing) {
1289         case MMC_TIMING_UHS_SDR12:
1290                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1291                 break;
1292         case MMC_TIMING_UHS_SDR25:
1293                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1294                 break;
1295         case MMC_TIMING_UHS_SDR50:
1296                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1297                 break;
1298         case MMC_TIMING_UHS_SDR104:
1299         case MMC_TIMING_MMC_HS200:
1300                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1301                 break;
1302         case MMC_TIMING_UHS_DDR50:
1303         case MMC_TIMING_MMC_DDR52:
1304                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1305                 break;
1306         case MMC_TIMING_MMC_HS400:
1307                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1308                 break;
1309         default:
1310                 pr_warn("%s: Invalid UHS-I mode selected\n",
1311                         mmc_hostname(host->mmc));
1312                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1313                 break;
1314         }
1315         return preset;
1316 }
1317
1318 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1319                    unsigned int *actual_clock)
1320 {
1321         int div = 0; /* Initialized for compiler warning */
1322         int real_div = div, clk_mul = 1;
1323         u16 clk = 0;
1324         bool switch_base_clk = false;
1325
1326         if (host->version >= SDHCI_SPEC_300) {
1327                 if (host->preset_enabled) {
1328                         u16 pre_val;
1329
1330                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1331                         pre_val = sdhci_get_preset_value(host);
1332                         div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1333                                 >> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1334                         if (host->clk_mul &&
1335                                 (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1336                                 clk = SDHCI_PROG_CLOCK_MODE;
1337                                 real_div = div + 1;
1338                                 clk_mul = host->clk_mul;
1339                         } else {
1340                                 real_div = max_t(int, 1, div << 1);
1341                         }
1342                         goto clock_set;
1343                 }
1344
1345                 /*
1346                  * Check if the Host Controller supports Programmable Clock
1347                  * Mode.
1348                  */
1349                 if (host->clk_mul) {
1350                         for (div = 1; div <= 1024; div++) {
1351                                 if ((host->max_clk * host->clk_mul / div)
1352                                         <= clock)
1353                                         break;
1354                         }
1355                         if ((host->max_clk * host->clk_mul / div) <= clock) {
1356                                 /*
1357                                  * Set Programmable Clock Mode in the Clock
1358                                  * Control register.
1359                                  */
1360                                 clk = SDHCI_PROG_CLOCK_MODE;
1361                                 real_div = div;
1362                                 clk_mul = host->clk_mul;
1363                                 div--;
1364                         } else {
1365                                 /*
1366                                  * Divisor can be too small to reach clock
1367                                  * speed requirement. Then use the base clock.
1368                                  */
1369                                 switch_base_clk = true;
1370                         }
1371                 }
1372
1373                 if (!host->clk_mul || switch_base_clk) {
1374                         /* Version 3.00 divisors must be a multiple of 2. */
1375                         if (host->max_clk <= clock)
1376                                 div = 1;
1377                         else {
1378                                 for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1379                                      div += 2) {
1380                                         if ((host->max_clk / div) <= clock)
1381                                                 break;
1382                                 }
1383                         }
1384                         real_div = div;
1385                         div >>= 1;
1386                         if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1387                                 && !div && host->max_clk <= 25000000)
1388                                 div = 1;
1389                 }
1390         } else {
1391                 /* Version 2.00 divisors must be a power of 2. */
1392                 for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1393                         if ((host->max_clk / div) <= clock)
1394                                 break;
1395                 }
1396                 real_div = div;
1397                 div >>= 1;
1398         }
1399
1400 clock_set:
1401         if (real_div)
1402                 *actual_clock = (host->max_clk * clk_mul) / real_div;
1403         clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1404         clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1405                 << SDHCI_DIVIDER_HI_SHIFT;
1406
1407         return clk;
1408 }
1409 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1410
1411 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1412 {
1413         ktime_t timeout;
1414
1415         clk |= SDHCI_CLOCK_INT_EN;
1416         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1417
1418         /* Wait max 20 ms */
1419         timeout = ktime_add_ms(ktime_get(), 20);
1420         while (!((clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL))
1421                 & SDHCI_CLOCK_INT_STABLE)) {
1422                 if (ktime_after(ktime_get(), timeout)) {
1423                         pr_err("%s: Internal clock never stabilised.\n",
1424                                mmc_hostname(host->mmc));
1425                         sdhci_dumpregs(host);
1426                         return;
1427                 }
1428                 udelay(10);
1429         }
1430
1431         clk |= SDHCI_CLOCK_CARD_EN;
1432         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1433 }
1434 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1435
1436 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1437 {
1438         u16 clk;
1439
1440         host->mmc->actual_clock = 0;
1441
1442         sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1443
1444         if (clock == 0)
1445                 return;
1446
1447         clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1448         sdhci_enable_clk(host, clk);
1449 }
1450 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1451
1452 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1453                                 unsigned short vdd)
1454 {
1455         struct mmc_host *mmc = host->mmc;
1456
1457         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1458
1459         if (mode != MMC_POWER_OFF)
1460                 sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1461         else
1462                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1463 }
1464
1465 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1466                            unsigned short vdd)
1467 {
1468         u8 pwr = 0;
1469
1470         if (mode != MMC_POWER_OFF) {
1471                 switch (1 << vdd) {
1472                 case MMC_VDD_165_195:
1473                 /*
1474                  * Without a regulator, SDHCI does not support 2.0v
1475                  * so we only get here if the driver deliberately
1476                  * added the 2.0v range to ocr_avail. Map it to 1.8v
1477                  * for the purpose of turning on the power.
1478                  */
1479                 case MMC_VDD_20_21:
1480                         pwr = SDHCI_POWER_180;
1481                         break;
1482                 case MMC_VDD_29_30:
1483                 case MMC_VDD_30_31:
1484                         pwr = SDHCI_POWER_300;
1485                         break;
1486                 case MMC_VDD_32_33:
1487                 case MMC_VDD_33_34:
1488                         pwr = SDHCI_POWER_330;
1489                         break;
1490                 default:
1491                         WARN(1, "%s: Invalid vdd %#x\n",
1492                              mmc_hostname(host->mmc), vdd);
1493                         break;
1494                 }
1495         }
1496
1497         if (host->pwr == pwr)
1498                 return;
1499
1500         host->pwr = pwr;
1501
1502         if (pwr == 0) {
1503                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1504                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1505                         sdhci_runtime_pm_bus_off(host);
1506         } else {
1507                 /*
1508                  * Spec says that we should clear the power reg before setting
1509                  * a new value. Some controllers don't seem to like this though.
1510                  */
1511                 if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1512                         sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1513
1514                 /*
1515                  * At least the Marvell CaFe chip gets confused if we set the
1516                  * voltage and set turn on power at the same time, so set the
1517                  * voltage first.
1518                  */
1519                 if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1520                         sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1521
1522                 pwr |= SDHCI_POWER_ON;
1523
1524                 sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1525
1526                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1527                         sdhci_runtime_pm_bus_on(host);
1528
1529                 /*
1530                  * Some controllers need an extra 10ms delay of 10ms before
1531                  * they can apply clock after applying power
1532                  */
1533                 if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1534                         mdelay(10);
1535         }
1536 }
1537 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1538
1539 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1540                      unsigned short vdd)
1541 {
1542         if (IS_ERR(host->mmc->supply.vmmc))
1543                 sdhci_set_power_noreg(host, mode, vdd);
1544         else
1545                 sdhci_set_power_reg(host, mode, vdd);
1546 }
1547 EXPORT_SYMBOL_GPL(sdhci_set_power);
1548
1549 /*****************************************************************************\
1550  *                                                                           *
1551  * MMC callbacks                                                             *
1552  *                                                                           *
1553 \*****************************************************************************/
1554
1555 static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1556 {
1557         struct sdhci_host *host;
1558         int present;
1559         unsigned long flags;
1560
1561         host = mmc_priv(mmc);
1562
1563         /* Firstly check card presence */
1564         present = mmc->ops->get_cd(mmc);
1565
1566         spin_lock_irqsave(&host->lock, flags);
1567
1568         sdhci_led_activate(host);
1569
1570         /*
1571          * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1572          * requests if Auto-CMD12 is enabled.
1573          */
1574         if (sdhci_auto_cmd12(host, mrq)) {
1575                 if (mrq->stop) {
1576                         mrq->data->stop = NULL;
1577                         mrq->stop = NULL;
1578                 }
1579         }
1580
1581         if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1582                 mrq->cmd->error = -ENOMEDIUM;
1583                 sdhci_finish_mrq(host, mrq);
1584         } else {
1585                 if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1586                         sdhci_send_command(host, mrq->sbc);
1587                 else
1588                         sdhci_send_command(host, mrq->cmd);
1589         }
1590
1591         mmiowb();
1592         spin_unlock_irqrestore(&host->lock, flags);
1593 }
1594
1595 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1596 {
1597         u8 ctrl;
1598
1599         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1600         if (width == MMC_BUS_WIDTH_8) {
1601                 ctrl &= ~SDHCI_CTRL_4BITBUS;
1602                 ctrl |= SDHCI_CTRL_8BITBUS;
1603         } else {
1604                 if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1605                         ctrl &= ~SDHCI_CTRL_8BITBUS;
1606                 if (width == MMC_BUS_WIDTH_4)
1607                         ctrl |= SDHCI_CTRL_4BITBUS;
1608                 else
1609                         ctrl &= ~SDHCI_CTRL_4BITBUS;
1610         }
1611         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1612 }
1613 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1614
1615 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1616 {
1617         u16 ctrl_2;
1618
1619         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1620         /* Select Bus Speed Mode for host */
1621         ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1622         if ((timing == MMC_TIMING_MMC_HS200) ||
1623             (timing == MMC_TIMING_UHS_SDR104))
1624                 ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1625         else if (timing == MMC_TIMING_UHS_SDR12)
1626                 ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1627         else if (timing == MMC_TIMING_UHS_SDR25)
1628                 ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1629         else if (timing == MMC_TIMING_UHS_SDR50)
1630                 ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1631         else if ((timing == MMC_TIMING_UHS_DDR50) ||
1632                  (timing == MMC_TIMING_MMC_DDR52))
1633                 ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1634         else if (timing == MMC_TIMING_MMC_HS400)
1635                 ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1636         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1637 }
1638 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1639
1640 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1641 {
1642         struct sdhci_host *host = mmc_priv(mmc);
1643         u8 ctrl;
1644
1645         if (ios->power_mode == MMC_POWER_UNDEFINED)
1646                 return;
1647
1648         if (host->flags & SDHCI_DEVICE_DEAD) {
1649                 if (!IS_ERR(mmc->supply.vmmc) &&
1650                     ios->power_mode == MMC_POWER_OFF)
1651                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1652                 return;
1653         }
1654
1655         /*
1656          * Reset the chip on each power off.
1657          * Should clear out any weird states.
1658          */
1659         if (ios->power_mode == MMC_POWER_OFF) {
1660                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1661                 sdhci_reinit(host);
1662         }
1663
1664         if (host->version >= SDHCI_SPEC_300 &&
1665                 (ios->power_mode == MMC_POWER_UP) &&
1666                 !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1667                 sdhci_enable_preset_value(host, false);
1668
1669         if (!ios->clock || ios->clock != host->clock) {
1670                 host->ops->set_clock(host, ios->clock);
1671                 host->clock = ios->clock;
1672
1673                 if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1674                     host->clock) {
1675                         host->timeout_clk = host->mmc->actual_clock ?
1676                                                 host->mmc->actual_clock / 1000 :
1677                                                 host->clock / 1000;
1678                         host->mmc->max_busy_timeout =
1679                                 host->ops->get_max_timeout_count ?
1680                                 host->ops->get_max_timeout_count(host) :
1681                                 1 << 27;
1682                         host->mmc->max_busy_timeout /= host->timeout_clk;
1683                 }
1684         }
1685
1686         if (host->ops->set_power)
1687                 host->ops->set_power(host, ios->power_mode, ios->vdd);
1688         else
1689                 sdhci_set_power(host, ios->power_mode, ios->vdd);
1690
1691         if (host->ops->platform_send_init_74_clocks)
1692                 host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1693
1694         host->ops->set_bus_width(host, ios->bus_width);
1695
1696         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1697
1698         if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1699                 if (ios->timing == MMC_TIMING_SD_HS ||
1700                      ios->timing == MMC_TIMING_MMC_HS ||
1701                      ios->timing == MMC_TIMING_MMC_HS400 ||
1702                      ios->timing == MMC_TIMING_MMC_HS200 ||
1703                      ios->timing == MMC_TIMING_MMC_DDR52 ||
1704                      ios->timing == MMC_TIMING_UHS_SDR50 ||
1705                      ios->timing == MMC_TIMING_UHS_SDR104 ||
1706                      ios->timing == MMC_TIMING_UHS_DDR50 ||
1707                      ios->timing == MMC_TIMING_UHS_SDR25)
1708                         ctrl |= SDHCI_CTRL_HISPD;
1709                 else
1710                         ctrl &= ~SDHCI_CTRL_HISPD;
1711         }
1712
1713         if (host->version >= SDHCI_SPEC_300) {
1714                 u16 clk, ctrl_2;
1715
1716                 if (!host->preset_enabled) {
1717                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1718                         /*
1719                          * We only need to set Driver Strength if the
1720                          * preset value enable is not set.
1721                          */
1722                         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1723                         ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1724                         if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1725                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1726                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1727                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1728                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1729                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1730                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1731                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1732                         else {
1733                                 pr_warn("%s: invalid driver type, default to driver type B\n",
1734                                         mmc_hostname(mmc));
1735                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1736                         }
1737
1738                         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1739                 } else {
1740                         /*
1741                          * According to SDHC Spec v3.00, if the Preset Value
1742                          * Enable in the Host Control 2 register is set, we
1743                          * need to reset SD Clock Enable before changing High
1744                          * Speed Enable to avoid generating clock gliches.
1745                          */
1746
1747                         /* Reset SD Clock Enable */
1748                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1749                         clk &= ~SDHCI_CLOCK_CARD_EN;
1750                         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1751
1752                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1753
1754                         /* Re-enable SD Clock */
1755                         host->ops->set_clock(host, host->clock);
1756                 }
1757
1758                 /* Reset SD Clock Enable */
1759                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1760                 clk &= ~SDHCI_CLOCK_CARD_EN;
1761                 sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1762
1763                 host->ops->set_uhs_signaling(host, ios->timing);
1764                 host->timing = ios->timing;
1765
1766                 if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1767                                 ((ios->timing == MMC_TIMING_UHS_SDR12) ||
1768                                  (ios->timing == MMC_TIMING_UHS_SDR25) ||
1769                                  (ios->timing == MMC_TIMING_UHS_SDR50) ||
1770                                  (ios->timing == MMC_TIMING_UHS_SDR104) ||
1771                                  (ios->timing == MMC_TIMING_UHS_DDR50) ||
1772                                  (ios->timing == MMC_TIMING_MMC_DDR52))) {
1773                         u16 preset;
1774
1775                         sdhci_enable_preset_value(host, true);
1776                         preset = sdhci_get_preset_value(host);
1777                         ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1778                                 >> SDHCI_PRESET_DRV_SHIFT;
1779                 }
1780
1781                 /* Re-enable SD Clock */
1782                 host->ops->set_clock(host, host->clock);
1783         } else
1784                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1785
1786         /*
1787          * Some (ENE) controllers go apeshit on some ios operation,
1788          * signalling timeout and CRC errors even on CMD0. Resetting
1789          * it on each ios seems to solve the problem.
1790          */
1791         if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
1792                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1793
1794         mmiowb();
1795 }
1796 EXPORT_SYMBOL_GPL(sdhci_set_ios);
1797
1798 static int sdhci_get_cd(struct mmc_host *mmc)
1799 {
1800         struct sdhci_host *host = mmc_priv(mmc);
1801         int gpio_cd = mmc_gpio_get_cd(mmc);
1802
1803         if (host->flags & SDHCI_DEVICE_DEAD)
1804                 return 0;
1805
1806         /* If nonremovable, assume that the card is always present. */
1807         if (!mmc_card_is_removable(host->mmc))
1808                 return 1;
1809
1810         /*
1811          * Try slot gpio detect, if defined it take precedence
1812          * over build in controller functionality
1813          */
1814         if (gpio_cd >= 0)
1815                 return !!gpio_cd;
1816
1817         /* If polling, assume that the card is always present. */
1818         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
1819                 return 1;
1820
1821         /* Host native card detect */
1822         return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1823 }
1824
1825 static int sdhci_check_ro(struct sdhci_host *host)
1826 {
1827         unsigned long flags;
1828         int is_readonly;
1829
1830         spin_lock_irqsave(&host->lock, flags);
1831
1832         if (host->flags & SDHCI_DEVICE_DEAD)
1833                 is_readonly = 0;
1834         else if (host->ops->get_ro)
1835                 is_readonly = host->ops->get_ro(host);
1836         else
1837                 is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
1838                                 & SDHCI_WRITE_PROTECT);
1839
1840         spin_unlock_irqrestore(&host->lock, flags);
1841
1842         /* This quirk needs to be replaced by a callback-function later */
1843         return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
1844                 !is_readonly : is_readonly;
1845 }
1846
1847 #define SAMPLE_COUNT    5
1848
1849 static int sdhci_get_ro(struct mmc_host *mmc)
1850 {
1851         struct sdhci_host *host = mmc_priv(mmc);
1852         int i, ro_count;
1853
1854         if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
1855                 return sdhci_check_ro(host);
1856
1857         ro_count = 0;
1858         for (i = 0; i < SAMPLE_COUNT; i++) {
1859                 if (sdhci_check_ro(host)) {
1860                         if (++ro_count > SAMPLE_COUNT / 2)
1861                                 return 1;
1862                 }
1863                 msleep(30);
1864         }
1865         return 0;
1866 }
1867
1868 static void sdhci_hw_reset(struct mmc_host *mmc)
1869 {
1870         struct sdhci_host *host = mmc_priv(mmc);
1871
1872         if (host->ops && host->ops->hw_reset)
1873                 host->ops->hw_reset(host);
1874 }
1875
1876 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
1877 {
1878         if (!(host->flags & SDHCI_DEVICE_DEAD)) {
1879                 if (enable)
1880                         host->ier |= SDHCI_INT_CARD_INT;
1881                 else
1882                         host->ier &= ~SDHCI_INT_CARD_INT;
1883
1884                 sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1885                 sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1886                 mmiowb();
1887         }
1888 }
1889
1890 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1891 {
1892         struct sdhci_host *host = mmc_priv(mmc);
1893         unsigned long flags;
1894
1895         if (enable)
1896                 pm_runtime_get_noresume(host->mmc->parent);
1897
1898         spin_lock_irqsave(&host->lock, flags);
1899         if (enable)
1900                 host->flags |= SDHCI_SDIO_IRQ_ENABLED;
1901         else
1902                 host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
1903
1904         sdhci_enable_sdio_irq_nolock(host, enable);
1905         spin_unlock_irqrestore(&host->lock, flags);
1906
1907         if (!enable)
1908                 pm_runtime_put_noidle(host->mmc->parent);
1909 }
1910 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
1911
1912 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
1913                                       struct mmc_ios *ios)
1914 {
1915         struct sdhci_host *host = mmc_priv(mmc);
1916         u16 ctrl;
1917         int ret;
1918
1919         /*
1920          * Signal Voltage Switching is only applicable for Host Controllers
1921          * v3.00 and above.
1922          */
1923         if (host->version < SDHCI_SPEC_300)
1924                 return 0;
1925
1926         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1927
1928         switch (ios->signal_voltage) {
1929         case MMC_SIGNAL_VOLTAGE_330:
1930                 if (!(host->flags & SDHCI_SIGNALING_330))
1931                         return -EINVAL;
1932                 /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
1933                 ctrl &= ~SDHCI_CTRL_VDD_180;
1934                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1935
1936                 if (!IS_ERR(mmc->supply.vqmmc)) {
1937                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1938                         if (ret) {
1939                                 pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
1940                                         mmc_hostname(mmc));
1941                                 return -EIO;
1942                         }
1943                 }
1944                 /* Wait for 5ms */
1945                 usleep_range(5000, 5500);
1946
1947                 /* 3.3V regulator output should be stable within 5 ms */
1948                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1949                 if (!(ctrl & SDHCI_CTRL_VDD_180))
1950                         return 0;
1951
1952                 pr_warn("%s: 3.3V regulator output did not became stable\n",
1953                         mmc_hostname(mmc));
1954
1955                 return -EAGAIN;
1956         case MMC_SIGNAL_VOLTAGE_180:
1957                 if (!(host->flags & SDHCI_SIGNALING_180))
1958                         return -EINVAL;
1959                 if (!IS_ERR(mmc->supply.vqmmc)) {
1960                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1961                         if (ret) {
1962                                 pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
1963                                         mmc_hostname(mmc));
1964                                 return -EIO;
1965                         }
1966                 }
1967
1968                 /*
1969                  * Enable 1.8V Signal Enable in the Host Control2
1970                  * register
1971                  */
1972                 ctrl |= SDHCI_CTRL_VDD_180;
1973                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1974
1975                 /* Some controller need to do more when switching */
1976                 if (host->ops->voltage_switch)
1977                         host->ops->voltage_switch(host);
1978
1979                 /* 1.8V regulator output should be stable within 5 ms */
1980                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1981                 if (ctrl & SDHCI_CTRL_VDD_180)
1982                         return 0;
1983
1984                 pr_warn("%s: 1.8V regulator output did not became stable\n",
1985                         mmc_hostname(mmc));
1986
1987                 return -EAGAIN;
1988         case MMC_SIGNAL_VOLTAGE_120:
1989                 if (!(host->flags & SDHCI_SIGNALING_120))
1990                         return -EINVAL;
1991                 if (!IS_ERR(mmc->supply.vqmmc)) {
1992                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1993                         if (ret) {
1994                                 pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
1995                                         mmc_hostname(mmc));
1996                                 return -EIO;
1997                         }
1998                 }
1999                 return 0;
2000         default:
2001                 /* No signal voltage switch required */
2002                 return 0;
2003         }
2004 }
2005 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2006
2007 static int sdhci_card_busy(struct mmc_host *mmc)
2008 {
2009         struct sdhci_host *host = mmc_priv(mmc);
2010         u32 present_state;
2011
2012         /* Check whether DAT[0] is 0 */
2013         present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2014
2015         return !(present_state & SDHCI_DATA_0_LVL_MASK);
2016 }
2017
2018 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2019 {
2020         struct sdhci_host *host = mmc_priv(mmc);
2021         unsigned long flags;
2022
2023         spin_lock_irqsave(&host->lock, flags);
2024         host->flags |= SDHCI_HS400_TUNING;
2025         spin_unlock_irqrestore(&host->lock, flags);
2026
2027         return 0;
2028 }
2029
2030 static void sdhci_start_tuning(struct sdhci_host *host)
2031 {
2032         u16 ctrl;
2033
2034         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2035         ctrl |= SDHCI_CTRL_EXEC_TUNING;
2036         if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2037                 ctrl |= SDHCI_CTRL_TUNED_CLK;
2038         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2039
2040         /*
2041          * As per the Host Controller spec v3.00, tuning command
2042          * generates Buffer Read Ready interrupt, so enable that.
2043          *
2044          * Note: The spec clearly says that when tuning sequence
2045          * is being performed, the controller does not generate
2046          * interrupts other than Buffer Read Ready interrupt. But
2047          * to make sure we don't hit a controller bug, we _only_
2048          * enable Buffer Read Ready interrupt here.
2049          */
2050         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2051         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2052 }
2053
2054 static void sdhci_end_tuning(struct sdhci_host *host)
2055 {
2056         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2057         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2058 }
2059
2060 static void sdhci_reset_tuning(struct sdhci_host *host)
2061 {
2062         u16 ctrl;
2063
2064         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2065         ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2066         ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2067         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2068 }
2069
2070 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2071 {
2072         sdhci_reset_tuning(host);
2073
2074         sdhci_do_reset(host, SDHCI_RESET_CMD);
2075         sdhci_do_reset(host, SDHCI_RESET_DATA);
2076
2077         sdhci_end_tuning(host);
2078
2079         mmc_abort_tuning(host->mmc, opcode);
2080 }
2081
2082 /*
2083  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2084  * tuning command does not have a data payload (or rather the hardware does it
2085  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2086  * interrupt setup is different to other commands and there is no timeout
2087  * interrupt so special handling is needed.
2088  */
2089 static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2090 {
2091         struct mmc_host *mmc = host->mmc;
2092         struct mmc_command cmd = {};
2093         struct mmc_request mrq = {};
2094         unsigned long flags;
2095         u32 b = host->sdma_boundary;
2096
2097         spin_lock_irqsave(&host->lock, flags);
2098
2099         cmd.opcode = opcode;
2100         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2101         cmd.mrq = &mrq;
2102
2103         mrq.cmd = &cmd;
2104         /*
2105          * In response to CMD19, the card sends 64 bytes of tuning
2106          * block to the Host Controller. So we set the block size
2107          * to 64 here.
2108          */
2109         if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2110             mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2111                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2112         else
2113                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2114
2115         /*
2116          * The tuning block is sent by the card to the host controller.
2117          * So we set the TRNS_READ bit in the Transfer Mode register.
2118          * This also takes care of setting DMA Enable and Multi Block
2119          * Select in the same register to 0.
2120          */
2121         sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2122
2123         sdhci_send_command(host, &cmd);
2124
2125         host->cmd = NULL;
2126
2127         sdhci_del_timer(host, &mrq);
2128
2129         host->tuning_done = 0;
2130
2131         mmiowb();
2132         spin_unlock_irqrestore(&host->lock, flags);
2133
2134         /* Wait for Buffer Read Ready interrupt */
2135         wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2136                            msecs_to_jiffies(50));
2137
2138 }
2139
2140 static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2141 {
2142         int i;
2143
2144         /*
2145          * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2146          * of loops reaches 40 times.
2147          */
2148         for (i = 0; i < MAX_TUNING_LOOP; i++) {
2149                 u16 ctrl;
2150
2151                 sdhci_send_tuning(host, opcode);
2152
2153                 if (!host->tuning_done) {
2154                         pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
2155                                 mmc_hostname(host->mmc));
2156                         sdhci_abort_tuning(host, opcode);
2157                         return;
2158                 }
2159
2160                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2161                 if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2162                         if (ctrl & SDHCI_CTRL_TUNED_CLK)
2163                                 return; /* Success! */
2164                         break;
2165                 }
2166
2167                 /* Spec does not require a delay between tuning cycles */
2168                 if (host->tuning_delay > 0)
2169                         mdelay(host->tuning_delay);
2170         }
2171
2172         pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2173                 mmc_hostname(host->mmc));
2174         sdhci_reset_tuning(host);
2175 }
2176
2177 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2178 {
2179         struct sdhci_host *host = mmc_priv(mmc);
2180         int err = 0;
2181         unsigned int tuning_count = 0;
2182         bool hs400_tuning;
2183
2184         hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2185
2186         if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2187                 tuning_count = host->tuning_count;
2188
2189         /*
2190          * The Host Controller needs tuning in case of SDR104 and DDR50
2191          * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2192          * the Capabilities register.
2193          * If the Host Controller supports the HS200 mode then the
2194          * tuning function has to be executed.
2195          */
2196         switch (host->timing) {
2197         /* HS400 tuning is done in HS200 mode */
2198         case MMC_TIMING_MMC_HS400:
2199                 err = -EINVAL;
2200                 goto out;
2201
2202         case MMC_TIMING_MMC_HS200:
2203                 /*
2204                  * Periodic re-tuning for HS400 is not expected to be needed, so
2205                  * disable it here.
2206                  */
2207                 if (hs400_tuning)
2208                         tuning_count = 0;
2209                 break;
2210
2211         case MMC_TIMING_UHS_SDR104:
2212         case MMC_TIMING_UHS_DDR50:
2213                 break;
2214
2215         case MMC_TIMING_UHS_SDR50:
2216                 if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2217                         break;
2218                 /* FALLTHROUGH */
2219
2220         default:
2221                 goto out;
2222         }
2223
2224         if (host->ops->platform_execute_tuning) {
2225                 err = host->ops->platform_execute_tuning(host, opcode);
2226                 goto out;
2227         }
2228
2229         host->mmc->retune_period = tuning_count;
2230
2231         if (host->tuning_delay < 0)
2232                 host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2233
2234         sdhci_start_tuning(host);
2235
2236         __sdhci_execute_tuning(host, opcode);
2237
2238         sdhci_end_tuning(host);
2239 out:
2240         host->flags &= ~SDHCI_HS400_TUNING;
2241
2242         return err;
2243 }
2244 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2245
2246 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2247 {
2248         /* Host Controller v3.00 defines preset value registers */
2249         if (host->version < SDHCI_SPEC_300)
2250                 return;
2251
2252         /*
2253          * We only enable or disable Preset Value if they are not already
2254          * enabled or disabled respectively. Otherwise, we bail out.
2255          */
2256         if (host->preset_enabled != enable) {
2257                 u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2258
2259                 if (enable)
2260                         ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2261                 else
2262                         ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2263
2264                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2265
2266                 if (enable)
2267                         host->flags |= SDHCI_PV_ENABLED;
2268                 else
2269                         host->flags &= ~SDHCI_PV_ENABLED;
2270
2271                 host->preset_enabled = enable;
2272         }
2273 }
2274
2275 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2276                                 int err)
2277 {
2278         struct sdhci_host *host = mmc_priv(mmc);
2279         struct mmc_data *data = mrq->data;
2280
2281         if (data->host_cookie != COOKIE_UNMAPPED)
2282                 dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2283                              mmc_get_dma_dir(data));
2284
2285         data->host_cookie = COOKIE_UNMAPPED;
2286 }
2287
2288 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2289 {
2290         struct sdhci_host *host = mmc_priv(mmc);
2291
2292         mrq->data->host_cookie = COOKIE_UNMAPPED;
2293
2294         /*
2295          * No pre-mapping in the pre hook if we're using the bounce buffer,
2296          * for that we would need two bounce buffers since one buffer is
2297          * in flight when this is getting called.
2298          */
2299         if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2300                 sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2301 }
2302
2303 static inline bool sdhci_has_requests(struct sdhci_host *host)
2304 {
2305         return host->cmd || host->data_cmd;
2306 }
2307
2308 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2309 {
2310         if (host->data_cmd) {
2311                 host->data_cmd->error = err;
2312                 sdhci_finish_mrq(host, host->data_cmd->mrq);
2313         }
2314
2315         if (host->cmd) {
2316                 host->cmd->error = err;
2317                 sdhci_finish_mrq(host, host->cmd->mrq);
2318         }
2319 }
2320
2321 static void sdhci_card_event(struct mmc_host *mmc)
2322 {
2323         struct sdhci_host *host = mmc_priv(mmc);
2324         unsigned long flags;
2325         int present;
2326
2327         /* First check if client has provided their own card event */
2328         if (host->ops->card_event)
2329                 host->ops->card_event(host);
2330
2331         present = mmc->ops->get_cd(mmc);
2332
2333         spin_lock_irqsave(&host->lock, flags);
2334
2335         /* Check sdhci_has_requests() first in case we are runtime suspended */
2336         if (sdhci_has_requests(host) && !present) {
2337                 pr_err("%s: Card removed during transfer!\n",
2338                         mmc_hostname(host->mmc));
2339                 pr_err("%s: Resetting controller.\n",
2340                         mmc_hostname(host->mmc));
2341
2342                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2343                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2344
2345                 sdhci_error_out_mrqs(host, -ENOMEDIUM);
2346         }
2347
2348         spin_unlock_irqrestore(&host->lock, flags);
2349 }
2350
2351 static const struct mmc_host_ops sdhci_ops = {
2352         .request        = sdhci_request,
2353         .post_req       = sdhci_post_req,
2354         .pre_req        = sdhci_pre_req,
2355         .set_ios        = sdhci_set_ios,
2356         .get_cd         = sdhci_get_cd,
2357         .get_ro         = sdhci_get_ro,
2358         .hw_reset       = sdhci_hw_reset,
2359         .enable_sdio_irq = sdhci_enable_sdio_irq,
2360         .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2361         .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2362         .execute_tuning                 = sdhci_execute_tuning,
2363         .card_event                     = sdhci_card_event,
2364         .card_busy      = sdhci_card_busy,
2365 };
2366
2367 /*****************************************************************************\
2368  *                                                                           *
2369  * Tasklets                                                                  *
2370  *                                                                           *
2371 \*****************************************************************************/
2372
2373 static bool sdhci_request_done(struct sdhci_host *host)
2374 {
2375         unsigned long flags;
2376         struct mmc_request *mrq;
2377         int i;
2378
2379         spin_lock_irqsave(&host->lock, flags);
2380
2381         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2382                 mrq = host->mrqs_done[i];
2383                 if (mrq)
2384                         break;
2385         }
2386
2387         if (!mrq) {
2388                 spin_unlock_irqrestore(&host->lock, flags);
2389                 return true;
2390         }
2391
2392         sdhci_del_timer(host, mrq);
2393
2394         /*
2395          * Always unmap the data buffers if they were mapped by
2396          * sdhci_prepare_data() whenever we finish with a request.
2397          * This avoids leaking DMA mappings on error.
2398          */
2399         if (host->flags & SDHCI_REQ_USE_DMA) {
2400                 struct mmc_data *data = mrq->data;
2401
2402                 if (data && data->host_cookie == COOKIE_MAPPED) {
2403                         if (host->bounce_buffer) {
2404                                 /*
2405                                  * On reads, copy the bounced data into the
2406                                  * sglist
2407                                  */
2408                                 if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2409                                         unsigned int length = data->bytes_xfered;
2410
2411                                         if (length > host->bounce_buffer_size) {
2412                                                 pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2413                                                        mmc_hostname(host->mmc),
2414                                                        host->bounce_buffer_size,
2415                                                        data->bytes_xfered);
2416                                                 /* Cap it down and continue */
2417                                                 length = host->bounce_buffer_size;
2418                                         }
2419                                         dma_sync_single_for_cpu(
2420                                                 host->mmc->parent,
2421                                                 host->bounce_addr,
2422                                                 host->bounce_buffer_size,
2423                                                 DMA_FROM_DEVICE);
2424                                         sg_copy_from_buffer(data->sg,
2425                                                 data->sg_len,
2426                                                 host->bounce_buffer,
2427                                                 length);
2428                                 } else {
2429                                         /* No copying, just switch ownership */
2430                                         dma_sync_single_for_cpu(
2431                                                 host->mmc->parent,
2432                                                 host->bounce_addr,
2433                                                 host->bounce_buffer_size,
2434                                                 mmc_get_dma_dir(data));
2435                                 }
2436                         } else {
2437                                 /* Unmap the raw data */
2438                                 dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2439                                              data->sg_len,
2440                                              mmc_get_dma_dir(data));
2441                         }
2442                         data->host_cookie = COOKIE_UNMAPPED;
2443                 }
2444         }
2445
2446         /*
2447          * The controller needs a reset of internal state machines
2448          * upon error conditions.
2449          */
2450         if (sdhci_needs_reset(host, mrq)) {
2451                 /*
2452                  * Do not finish until command and data lines are available for
2453                  * reset. Note there can only be one other mrq, so it cannot
2454                  * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2455                  * would both be null.
2456                  */
2457                 if (host->cmd || host->data_cmd) {
2458                         spin_unlock_irqrestore(&host->lock, flags);
2459                         return true;
2460                 }
2461
2462                 /* Some controllers need this kick or reset won't work here */
2463                 if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2464                         /* This is to force an update */
2465                         host->ops->set_clock(host, host->clock);
2466
2467                 /* Spec says we should do both at the same time, but Ricoh
2468                    controllers do not like that. */
2469                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2470                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2471
2472                 host->pending_reset = false;
2473         }
2474
2475         if (!sdhci_has_requests(host))
2476                 sdhci_led_deactivate(host);
2477
2478         host->mrqs_done[i] = NULL;
2479
2480         mmiowb();
2481         spin_unlock_irqrestore(&host->lock, flags);
2482
2483         mmc_request_done(host->mmc, mrq);
2484
2485         return false;
2486 }
2487
2488 static void sdhci_tasklet_finish(unsigned long param)
2489 {
2490         struct sdhci_host *host = (struct sdhci_host *)param;
2491
2492         while (!sdhci_request_done(host))
2493                 ;
2494 }
2495
2496 static void sdhci_timeout_timer(struct timer_list *t)
2497 {
2498         struct sdhci_host *host;
2499         unsigned long flags;
2500
2501         host = from_timer(host, t, timer);
2502
2503         spin_lock_irqsave(&host->lock, flags);
2504
2505         if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2506                 pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2507                        mmc_hostname(host->mmc));
2508                 sdhci_dumpregs(host);
2509
2510                 host->cmd->error = -ETIMEDOUT;
2511                 sdhci_finish_mrq(host, host->cmd->mrq);
2512         }
2513
2514         mmiowb();
2515         spin_unlock_irqrestore(&host->lock, flags);
2516 }
2517
2518 static void sdhci_timeout_data_timer(struct timer_list *t)
2519 {
2520         struct sdhci_host *host;
2521         unsigned long flags;
2522
2523         host = from_timer(host, t, data_timer);
2524
2525         spin_lock_irqsave(&host->lock, flags);
2526
2527         if (host->data || host->data_cmd ||
2528             (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2529                 pr_err("%s: Timeout waiting for hardware interrupt.\n",
2530                        mmc_hostname(host->mmc));
2531                 sdhci_dumpregs(host);
2532
2533                 if (host->data) {
2534                         host->data->error = -ETIMEDOUT;
2535                         sdhci_finish_data(host);
2536                 } else if (host->data_cmd) {
2537                         host->data_cmd->error = -ETIMEDOUT;
2538                         sdhci_finish_mrq(host, host->data_cmd->mrq);
2539                 } else {
2540                         host->cmd->error = -ETIMEDOUT;
2541                         sdhci_finish_mrq(host, host->cmd->mrq);
2542                 }
2543         }
2544
2545         mmiowb();
2546         spin_unlock_irqrestore(&host->lock, flags);
2547 }
2548
2549 /*****************************************************************************\
2550  *                                                                           *
2551  * Interrupt handling                                                        *
2552  *                                                                           *
2553 \*****************************************************************************/
2554
2555 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask)
2556 {
2557         if (!host->cmd) {
2558                 /*
2559                  * SDHCI recovers from errors by resetting the cmd and data
2560                  * circuits.  Until that is done, there very well might be more
2561                  * interrupts, so ignore them in that case.
2562                  */
2563                 if (host->pending_reset)
2564                         return;
2565                 pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2566                        mmc_hostname(host->mmc), (unsigned)intmask);
2567                 sdhci_dumpregs(host);
2568                 return;
2569         }
2570
2571         if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2572                        SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2573                 if (intmask & SDHCI_INT_TIMEOUT)
2574                         host->cmd->error = -ETIMEDOUT;
2575                 else
2576                         host->cmd->error = -EILSEQ;
2577
2578                 /*
2579                  * If this command initiates a data phase and a response
2580                  * CRC error is signalled, the card can start transferring
2581                  * data - the card may have received the command without
2582                  * error.  We must not terminate the mmc_request early.
2583                  *
2584                  * If the card did not receive the command or returned an
2585                  * error which prevented it sending data, the data phase
2586                  * will time out.
2587                  */
2588                 if (host->cmd->data &&
2589                     (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2590                      SDHCI_INT_CRC) {
2591                         host->cmd = NULL;
2592                         return;
2593                 }
2594
2595                 sdhci_finish_mrq(host, host->cmd->mrq);
2596                 return;
2597         }
2598
2599         if (intmask & SDHCI_INT_RESPONSE)
2600                 sdhci_finish_command(host);
2601 }
2602
2603 static void sdhci_adma_show_error(struct sdhci_host *host)
2604 {
2605         void *desc = host->adma_table;
2606
2607         sdhci_dumpregs(host);
2608
2609         while (true) {
2610                 struct sdhci_adma2_64_desc *dma_desc = desc;
2611
2612                 if (host->flags & SDHCI_USE_64_BIT_DMA)
2613                         DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2614                             desc, le32_to_cpu(dma_desc->addr_hi),
2615                             le32_to_cpu(dma_desc->addr_lo),
2616                             le16_to_cpu(dma_desc->len),
2617                             le16_to_cpu(dma_desc->cmd));
2618                 else
2619                         DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2620                             desc, le32_to_cpu(dma_desc->addr_lo),
2621                             le16_to_cpu(dma_desc->len),
2622                             le16_to_cpu(dma_desc->cmd));
2623
2624                 desc += host->desc_sz;
2625
2626                 if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2627                         break;
2628         }
2629 }
2630
2631 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2632 {
2633         u32 command;
2634
2635         /* CMD19 generates _only_ Buffer Read Ready interrupt */
2636         if (intmask & SDHCI_INT_DATA_AVAIL) {
2637                 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2638                 if (command == MMC_SEND_TUNING_BLOCK ||
2639                     command == MMC_SEND_TUNING_BLOCK_HS200) {
2640                         host->tuning_done = 1;
2641                         wake_up(&host->buf_ready_int);
2642                         return;
2643                 }
2644         }
2645
2646         if (!host->data) {
2647                 struct mmc_command *data_cmd = host->data_cmd;
2648
2649                 /*
2650                  * The "data complete" interrupt is also used to
2651                  * indicate that a busy state has ended. See comment
2652                  * above in sdhci_cmd_irq().
2653                  */
2654                 if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2655                         if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2656                                 host->data_cmd = NULL;
2657                                 data_cmd->error = -ETIMEDOUT;
2658                                 sdhci_finish_mrq(host, data_cmd->mrq);
2659                                 return;
2660                         }
2661                         if (intmask & SDHCI_INT_DATA_END) {
2662                                 host->data_cmd = NULL;
2663                                 /*
2664                                  * Some cards handle busy-end interrupt
2665                                  * before the command completed, so make
2666                                  * sure we do things in the proper order.
2667                                  */
2668                                 if (host->cmd == data_cmd)
2669                                         return;
2670
2671                                 sdhci_finish_mrq(host, data_cmd->mrq);
2672                                 return;
2673                         }
2674                 }
2675
2676                 /*
2677                  * SDHCI recovers from errors by resetting the cmd and data
2678                  * circuits. Until that is done, there very well might be more
2679                  * interrupts, so ignore them in that case.
2680                  */
2681                 if (host->pending_reset)
2682                         return;
2683
2684                 pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2685                        mmc_hostname(host->mmc), (unsigned)intmask);
2686                 sdhci_dumpregs(host);
2687
2688                 return;
2689         }
2690
2691         if (intmask & SDHCI_INT_DATA_TIMEOUT)
2692                 host->data->error = -ETIMEDOUT;
2693         else if (intmask & SDHCI_INT_DATA_END_BIT)
2694                 host->data->error = -EILSEQ;
2695         else if ((intmask & SDHCI_INT_DATA_CRC) &&
2696                 SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2697                         != MMC_BUS_TEST_R)
2698                 host->data->error = -EILSEQ;
2699         else if (intmask & SDHCI_INT_ADMA_ERROR) {
2700                 pr_err("%s: ADMA error\n", mmc_hostname(host->mmc));
2701                 sdhci_adma_show_error(host);
2702                 host->data->error = -EIO;
2703                 if (host->ops->adma_workaround)
2704                         host->ops->adma_workaround(host, intmask);
2705         }
2706
2707         if (host->data->error)
2708                 sdhci_finish_data(host);
2709         else {
2710                 if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2711                         sdhci_transfer_pio(host);
2712
2713                 /*
2714                  * We currently don't do anything fancy with DMA
2715                  * boundaries, but as we can't disable the feature
2716                  * we need to at least restart the transfer.
2717                  *
2718                  * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2719                  * should return a valid address to continue from, but as
2720                  * some controllers are faulty, don't trust them.
2721                  */
2722                 if (intmask & SDHCI_INT_DMA_END) {
2723                         u32 dmastart, dmanow;
2724
2725                         dmastart = sdhci_sdma_address(host);
2726                         dmanow = dmastart + host->data->bytes_xfered;
2727                         /*
2728                          * Force update to the next DMA block boundary.
2729                          */
2730                         dmanow = (dmanow &
2731                                 ~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2732                                 SDHCI_DEFAULT_BOUNDARY_SIZE;
2733                         host->data->bytes_xfered = dmanow - dmastart;
2734                         DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n",
2735                             dmastart, host->data->bytes_xfered, dmanow);
2736                         sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS);
2737                 }
2738
2739                 if (intmask & SDHCI_INT_DATA_END) {
2740                         if (host->cmd == host->data_cmd) {
2741                                 /*
2742                                  * Data managed to finish before the
2743                                  * command completed. Make sure we do
2744                                  * things in the proper order.
2745                                  */
2746                                 host->data_early = 1;
2747                         } else {
2748                                 sdhci_finish_data(host);
2749                         }
2750                 }
2751         }
2752 }
2753
2754 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2755 {
2756         irqreturn_t result = IRQ_NONE;
2757         struct sdhci_host *host = dev_id;
2758         u32 intmask, mask, unexpected = 0;
2759         int max_loops = 16;
2760
2761         spin_lock(&host->lock);
2762
2763         if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
2764                 spin_unlock(&host->lock);
2765                 return IRQ_NONE;
2766         }
2767
2768         intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2769         if (!intmask || intmask == 0xffffffff) {
2770                 result = IRQ_NONE;
2771                 goto out;
2772         }
2773
2774         do {
2775                 DBG("IRQ status 0x%08x\n", intmask);
2776
2777                 if (host->ops->irq) {
2778                         intmask = host->ops->irq(host, intmask);
2779                         if (!intmask)
2780                                 goto cont;
2781                 }
2782
2783                 /* Clear selected interrupts. */
2784                 mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2785                                   SDHCI_INT_BUS_POWER);
2786                 sdhci_writel(host, mask, SDHCI_INT_STATUS);
2787
2788                 if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2789                         u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
2790                                       SDHCI_CARD_PRESENT;
2791
2792                         /*
2793                          * There is a observation on i.mx esdhc.  INSERT
2794                          * bit will be immediately set again when it gets
2795                          * cleared, if a card is inserted.  We have to mask
2796                          * the irq to prevent interrupt storm which will
2797                          * freeze the system.  And the REMOVE gets the
2798                          * same situation.
2799                          *
2800                          * More testing are needed here to ensure it works
2801                          * for other platforms though.
2802                          */
2803                         host->ier &= ~(SDHCI_INT_CARD_INSERT |
2804                                        SDHCI_INT_CARD_REMOVE);
2805                         host->ier |= present ? SDHCI_INT_CARD_REMOVE :
2806                                                SDHCI_INT_CARD_INSERT;
2807                         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2808                         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2809
2810                         sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
2811                                      SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
2812
2813                         host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
2814                                                        SDHCI_INT_CARD_REMOVE);
2815                         result = IRQ_WAKE_THREAD;
2816                 }
2817
2818                 if (intmask & SDHCI_INT_CMD_MASK)
2819                         sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK);
2820
2821                 if (intmask & SDHCI_INT_DATA_MASK)
2822                         sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
2823
2824                 if (intmask & SDHCI_INT_BUS_POWER)
2825                         pr_err("%s: Card is consuming too much power!\n",
2826                                 mmc_hostname(host->mmc));
2827
2828                 if (intmask & SDHCI_INT_RETUNE)
2829                         mmc_retune_needed(host->mmc);
2830
2831                 if ((intmask & SDHCI_INT_CARD_INT) &&
2832                     (host->ier & SDHCI_INT_CARD_INT)) {
2833                         sdhci_enable_sdio_irq_nolock(host, false);
2834                         host->thread_isr |= SDHCI_INT_CARD_INT;
2835                         result = IRQ_WAKE_THREAD;
2836                 }
2837
2838                 intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2839                              SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2840                              SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
2841                              SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
2842
2843                 if (intmask) {
2844                         unexpected |= intmask;
2845                         sdhci_writel(host, intmask, SDHCI_INT_STATUS);
2846                 }
2847 cont:
2848                 if (result == IRQ_NONE)
2849                         result = IRQ_HANDLED;
2850
2851                 intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2852         } while (intmask && --max_loops);
2853 out:
2854         spin_unlock(&host->lock);
2855
2856         if (unexpected) {
2857                 pr_err("%s: Unexpected interrupt 0x%08x.\n",
2858                            mmc_hostname(host->mmc), unexpected);
2859                 sdhci_dumpregs(host);
2860         }
2861
2862         return result;
2863 }
2864
2865 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
2866 {
2867         struct sdhci_host *host = dev_id;
2868         unsigned long flags;
2869         u32 isr;
2870
2871         spin_lock_irqsave(&host->lock, flags);
2872         isr = host->thread_isr;
2873         host->thread_isr = 0;
2874         spin_unlock_irqrestore(&host->lock, flags);
2875
2876         if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2877                 struct mmc_host *mmc = host->mmc;
2878
2879                 mmc->ops->card_event(mmc);
2880                 mmc_detect_change(mmc, msecs_to_jiffies(200));
2881         }
2882
2883         if (isr & SDHCI_INT_CARD_INT) {
2884                 sdio_run_irqs(host->mmc);
2885
2886                 spin_lock_irqsave(&host->lock, flags);
2887                 if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
2888                         sdhci_enable_sdio_irq_nolock(host, true);
2889                 spin_unlock_irqrestore(&host->lock, flags);
2890         }
2891
2892         return isr ? IRQ_HANDLED : IRQ_NONE;
2893 }
2894
2895 /*****************************************************************************\
2896  *                                                                           *
2897  * Suspend/resume                                                            *
2898  *                                                                           *
2899 \*****************************************************************************/
2900
2901 #ifdef CONFIG_PM
2902
2903 static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
2904 {
2905         return mmc_card_is_removable(host->mmc) &&
2906                !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
2907                !mmc_can_gpio_cd(host->mmc);
2908 }
2909
2910 /*
2911  * To enable wakeup events, the corresponding events have to be enabled in
2912  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
2913  * Table' in the SD Host Controller Standard Specification.
2914  * It is useless to restore SDHCI_INT_ENABLE state in
2915  * sdhci_disable_irq_wakeups() since it will be set by
2916  * sdhci_enable_card_detection() or sdhci_init().
2917  */
2918 static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
2919 {
2920         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
2921                   SDHCI_WAKE_ON_INT;
2922         u32 irq_val = 0;
2923         u8 wake_val = 0;
2924         u8 val;
2925
2926         if (sdhci_cd_irq_can_wakeup(host)) {
2927                 wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
2928                 irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
2929         }
2930
2931         if (mmc_card_wake_sdio_irq(host->mmc)) {
2932                 wake_val |= SDHCI_WAKE_ON_INT;
2933                 irq_val |= SDHCI_INT_CARD_INT;
2934         }
2935
2936         if (!irq_val)
2937                 return false;
2938
2939         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2940         val &= ~mask;
2941         val |= wake_val;
2942         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2943
2944         sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
2945
2946         host->irq_wake_enabled = !enable_irq_wake(host->irq);
2947
2948         return host->irq_wake_enabled;
2949 }
2950
2951 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
2952 {
2953         u8 val;
2954         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
2955                         | SDHCI_WAKE_ON_INT;
2956
2957         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2958         val &= ~mask;
2959         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2960
2961         disable_irq_wake(host->irq);
2962
2963         host->irq_wake_enabled = false;
2964 }
2965
2966 int sdhci_suspend_host(struct sdhci_host *host)
2967 {
2968         sdhci_disable_card_detection(host);
2969
2970         mmc_retune_timer_stop(host->mmc);
2971
2972         if (!device_may_wakeup(mmc_dev(host->mmc)) ||
2973             !sdhci_enable_irq_wakeups(host)) {
2974                 host->ier = 0;
2975                 sdhci_writel(host, 0, SDHCI_INT_ENABLE);
2976                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
2977                 free_irq(host->irq, host);
2978         }
2979
2980         return 0;
2981 }
2982
2983 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
2984
2985 int sdhci_resume_host(struct sdhci_host *host)
2986 {
2987         struct mmc_host *mmc = host->mmc;
2988         int ret = 0;
2989
2990         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
2991                 if (host->ops->enable_dma)
2992                         host->ops->enable_dma(host);
2993         }
2994
2995         if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
2996             (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
2997                 /* Card keeps power but host controller does not */
2998                 sdhci_init(host, 0);
2999                 host->pwr = 0;
3000                 host->clock = 0;
3001                 mmc->ops->set_ios(mmc, &mmc->ios);
3002         } else {
3003                 sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3004                 mmiowb();
3005         }
3006
3007         if (host->irq_wake_enabled) {
3008                 sdhci_disable_irq_wakeups(host);
3009         } else {
3010                 ret = request_threaded_irq(host->irq, sdhci_irq,
3011                                            sdhci_thread_irq, IRQF_SHARED,
3012                                            mmc_hostname(host->mmc), host);
3013                 if (ret)
3014                         return ret;
3015         }
3016
3017         sdhci_enable_card_detection(host);
3018
3019         return ret;
3020 }
3021
3022 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3023
3024 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3025 {
3026         unsigned long flags;
3027
3028         mmc_retune_timer_stop(host->mmc);
3029
3030         spin_lock_irqsave(&host->lock, flags);
3031         host->ier &= SDHCI_INT_CARD_INT;
3032         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3033         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3034         spin_unlock_irqrestore(&host->lock, flags);
3035
3036         synchronize_hardirq(host->irq);
3037
3038         spin_lock_irqsave(&host->lock, flags);
3039         host->runtime_suspended = true;
3040         spin_unlock_irqrestore(&host->lock, flags);
3041
3042         return 0;
3043 }
3044 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3045
3046 int sdhci_runtime_resume_host(struct sdhci_host *host)
3047 {
3048         struct mmc_host *mmc = host->mmc;
3049         unsigned long flags;
3050         int host_flags = host->flags;
3051
3052         if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3053                 if (host->ops->enable_dma)
3054                         host->ops->enable_dma(host);
3055         }
3056
3057         sdhci_init(host, 0);
3058
3059         if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3060             mmc->ios.power_mode != MMC_POWER_OFF) {
3061                 /* Force clock and power re-program */
3062                 host->pwr = 0;
3063                 host->clock = 0;
3064                 mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3065                 mmc->ops->set_ios(mmc, &mmc->ios);
3066
3067                 if ((host_flags & SDHCI_PV_ENABLED) &&
3068                     !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3069                         spin_lock_irqsave(&host->lock, flags);
3070                         sdhci_enable_preset_value(host, true);
3071                         spin_unlock_irqrestore(&host->lock, flags);
3072                 }
3073
3074                 if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3075                     mmc->ops->hs400_enhanced_strobe)
3076                         mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3077         }
3078
3079         spin_lock_irqsave(&host->lock, flags);
3080
3081         host->runtime_suspended = false;
3082
3083         /* Enable SDIO IRQ */
3084         if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3085                 sdhci_enable_sdio_irq_nolock(host, true);
3086
3087         /* Enable Card Detection */
3088         sdhci_enable_card_detection(host);
3089
3090         spin_unlock_irqrestore(&host->lock, flags);
3091
3092         return 0;
3093 }
3094 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3095
3096 #endif /* CONFIG_PM */
3097
3098 /*****************************************************************************\
3099  *                                                                           *
3100  * Command Queue Engine (CQE) helpers                                        *
3101  *                                                                           *
3102 \*****************************************************************************/
3103
3104 void sdhci_cqe_enable(struct mmc_host *mmc)
3105 {
3106         struct sdhci_host *host = mmc_priv(mmc);
3107         unsigned long flags;
3108         u8 ctrl;
3109
3110         spin_lock_irqsave(&host->lock, flags);
3111
3112         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3113         ctrl &= ~SDHCI_CTRL_DMA_MASK;
3114         if (host->flags & SDHCI_USE_64_BIT_DMA)
3115                 ctrl |= SDHCI_CTRL_ADMA64;
3116         else
3117                 ctrl |= SDHCI_CTRL_ADMA32;
3118         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3119
3120         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3121                      SDHCI_BLOCK_SIZE);
3122
3123         /* Set maximum timeout */
3124         sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
3125
3126         host->ier = host->cqe_ier;
3127
3128         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3129         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3130
3131         host->cqe_on = true;
3132
3133         pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3134                  mmc_hostname(mmc), host->ier,
3135                  sdhci_readl(host, SDHCI_INT_STATUS));
3136
3137         mmiowb();
3138         spin_unlock_irqrestore(&host->lock, flags);
3139 }
3140 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3141
3142 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3143 {
3144         struct sdhci_host *host = mmc_priv(mmc);
3145         unsigned long flags;
3146
3147         spin_lock_irqsave(&host->lock, flags);
3148
3149         sdhci_set_default_irqs(host);
3150
3151         host->cqe_on = false;
3152
3153         if (recovery) {
3154                 sdhci_do_reset(host, SDHCI_RESET_CMD);
3155                 sdhci_do_reset(host, SDHCI_RESET_DATA);
3156         }
3157
3158         pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3159                  mmc_hostname(mmc), host->ier,
3160                  sdhci_readl(host, SDHCI_INT_STATUS));
3161
3162         mmiowb();
3163         spin_unlock_irqrestore(&host->lock, flags);
3164 }
3165 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3166
3167 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3168                    int *data_error)
3169 {
3170         u32 mask;
3171
3172         if (!host->cqe_on)
3173                 return false;
3174
3175         if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3176                 *cmd_error = -EILSEQ;
3177         else if (intmask & SDHCI_INT_TIMEOUT)
3178                 *cmd_error = -ETIMEDOUT;
3179         else
3180                 *cmd_error = 0;
3181
3182         if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3183                 *data_error = -EILSEQ;
3184         else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3185                 *data_error = -ETIMEDOUT;
3186         else if (intmask & SDHCI_INT_ADMA_ERROR)
3187                 *data_error = -EIO;
3188         else
3189                 *data_error = 0;
3190
3191         /* Clear selected interrupts. */
3192         mask = intmask & host->cqe_ier;
3193         sdhci_writel(host, mask, SDHCI_INT_STATUS);
3194
3195         if (intmask & SDHCI_INT_BUS_POWER)
3196                 pr_err("%s: Card is consuming too much power!\n",
3197                        mmc_hostname(host->mmc));
3198
3199         intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3200         if (intmask) {
3201                 sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3202                 pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3203                        mmc_hostname(host->mmc), intmask);
3204                 sdhci_dumpregs(host);
3205         }
3206
3207         return true;
3208 }
3209 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3210
3211 /*****************************************************************************\
3212  *                                                                           *
3213  * Device allocation/registration                                            *
3214  *                                                                           *
3215 \*****************************************************************************/
3216
3217 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3218         size_t priv_size)
3219 {
3220         struct mmc_host *mmc;
3221         struct sdhci_host *host;
3222
3223         WARN_ON(dev == NULL);
3224
3225         mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3226         if (!mmc)
3227                 return ERR_PTR(-ENOMEM);
3228
3229         host = mmc_priv(mmc);
3230         host->mmc = mmc;
3231         host->mmc_host_ops = sdhci_ops;
3232         mmc->ops = &host->mmc_host_ops;
3233
3234         host->flags = SDHCI_SIGNALING_330;
3235
3236         host->cqe_ier     = SDHCI_CQE_INT_MASK;
3237         host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3238
3239         host->tuning_delay = -1;
3240
3241         host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3242
3243         return host;
3244 }
3245
3246 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3247
3248 static int sdhci_set_dma_mask(struct sdhci_host *host)
3249 {
3250         struct mmc_host *mmc = host->mmc;
3251         struct device *dev = mmc_dev(mmc);
3252         int ret = -EINVAL;
3253
3254         if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3255                 host->flags &= ~SDHCI_USE_64_BIT_DMA;
3256
3257         /* Try 64-bit mask if hardware is capable  of it */
3258         if (host->flags & SDHCI_USE_64_BIT_DMA) {
3259                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3260                 if (ret) {
3261                         pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3262                                 mmc_hostname(mmc));
3263                         host->flags &= ~SDHCI_USE_64_BIT_DMA;
3264                 }
3265         }
3266
3267         /* 32-bit mask as default & fallback */
3268         if (ret) {
3269                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3270                 if (ret)
3271                         pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3272                                 mmc_hostname(mmc));
3273         }
3274
3275         return ret;
3276 }
3277
3278 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3279 {
3280         u16 v;
3281         u64 dt_caps_mask = 0;
3282         u64 dt_caps = 0;
3283
3284         if (host->read_caps)
3285                 return;
3286
3287         host->read_caps = true;
3288
3289         if (debug_quirks)
3290                 host->quirks = debug_quirks;
3291
3292         if (debug_quirks2)
3293                 host->quirks2 = debug_quirks2;
3294
3295         sdhci_do_reset(host, SDHCI_RESET_ALL);
3296
3297         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3298                              "sdhci-caps-mask", &dt_caps_mask);
3299         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3300                              "sdhci-caps", &dt_caps);
3301
3302         v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3303         host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3304
3305         if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3306                 return;
3307
3308         if (caps) {
3309                 host->caps = *caps;
3310         } else {
3311                 host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3312                 host->caps &= ~lower_32_bits(dt_caps_mask);
3313                 host->caps |= lower_32_bits(dt_caps);
3314         }
3315
3316         if (host->version < SDHCI_SPEC_300)
3317                 return;
3318
3319         if (caps1) {
3320                 host->caps1 = *caps1;
3321         } else {
3322                 host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3323                 host->caps1 &= ~upper_32_bits(dt_caps_mask);
3324                 host->caps1 |= upper_32_bits(dt_caps);
3325         }
3326 }
3327 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3328
3329 static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3330 {
3331         struct mmc_host *mmc = host->mmc;
3332         unsigned int max_blocks;
3333         unsigned int bounce_size;
3334         int ret;
3335
3336         /*
3337          * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3338          * has diminishing returns, this is probably because SD/MMC
3339          * cards are usually optimized to handle this size of requests.
3340          */
3341         bounce_size = SZ_64K;
3342         /*
3343          * Adjust downwards to maximum request size if this is less
3344          * than our segment size, else hammer down the maximum
3345          * request size to the maximum buffer size.
3346          */
3347         if (mmc->max_req_size < bounce_size)
3348                 bounce_size = mmc->max_req_size;
3349         max_blocks = bounce_size / 512;
3350
3351         /*
3352          * When we just support one segment, we can get significant
3353          * speedups by the help of a bounce buffer to group scattered
3354          * reads/writes together.
3355          */
3356         host->bounce_buffer = devm_kmalloc(mmc->parent,
3357                                            bounce_size,
3358                                            GFP_KERNEL);
3359         if (!host->bounce_buffer) {
3360                 pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3361                        mmc_hostname(mmc),
3362                        bounce_size);
3363                 /*
3364                  * Exiting with zero here makes sure we proceed with
3365                  * mmc->max_segs == 1.
3366                  */
3367                 return 0;
3368         }
3369
3370         host->bounce_addr = dma_map_single(mmc->parent,
3371                                            host->bounce_buffer,
3372                                            bounce_size,
3373                                            DMA_BIDIRECTIONAL);
3374         ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3375         if (ret)
3376                 /* Again fall back to max_segs == 1 */
3377                 return 0;
3378         host->bounce_buffer_size = bounce_size;
3379
3380         /* Lie about this since we're bouncing */
3381         mmc->max_segs = max_blocks;
3382         mmc->max_seg_size = bounce_size;
3383         mmc->max_req_size = bounce_size;
3384
3385         pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3386                 mmc_hostname(mmc), max_blocks, bounce_size);
3387
3388         return 0;
3389 }
3390
3391 int sdhci_setup_host(struct sdhci_host *host)
3392 {
3393         struct mmc_host *mmc;
3394         u32 max_current_caps;
3395         unsigned int ocr_avail;
3396         unsigned int override_timeout_clk;
3397         u32 max_clk;
3398         int ret;
3399
3400         WARN_ON(host == NULL);
3401         if (host == NULL)
3402                 return -EINVAL;
3403
3404         mmc = host->mmc;
3405
3406         /*
3407          * If there are external regulators, get them. Note this must be done
3408          * early before resetting the host and reading the capabilities so that
3409          * the host can take the appropriate action if regulators are not
3410          * available.
3411          */
3412         ret = mmc_regulator_get_supply(mmc);
3413         if (ret)
3414                 return ret;
3415
3416         DBG("Version:   0x%08x | Present:  0x%08x\n",
3417             sdhci_readw(host, SDHCI_HOST_VERSION),
3418             sdhci_readl(host, SDHCI_PRESENT_STATE));
3419         DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3420             sdhci_readl(host, SDHCI_CAPABILITIES),
3421             sdhci_readl(host, SDHCI_CAPABILITIES_1));
3422
3423         sdhci_read_caps(host);
3424
3425         override_timeout_clk = host->timeout_clk;
3426
3427         if (host->version > SDHCI_SPEC_300) {
3428                 pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3429                        mmc_hostname(mmc), host->version);
3430         }
3431
3432         if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3433                 host->flags |= SDHCI_USE_SDMA;
3434         else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3435                 DBG("Controller doesn't have SDMA capability\n");
3436         else
3437                 host->flags |= SDHCI_USE_SDMA;
3438
3439         if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3440                 (host->flags & SDHCI_USE_SDMA)) {
3441                 DBG("Disabling DMA as it is marked broken\n");
3442                 host->flags &= ~SDHCI_USE_SDMA;
3443         }
3444
3445         if ((host->version >= SDHCI_SPEC_200) &&
3446                 (host->caps & SDHCI_CAN_DO_ADMA2))
3447                 host->flags |= SDHCI_USE_ADMA;
3448
3449         if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3450                 (host->flags & SDHCI_USE_ADMA)) {
3451                 DBG("Disabling ADMA as it is marked broken\n");
3452                 host->flags &= ~SDHCI_USE_ADMA;
3453         }
3454
3455         /*
3456          * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3457          * and *must* do 64-bit DMA.  A driver has the opportunity to change
3458          * that during the first call to ->enable_dma().  Similarly
3459          * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3460          * implement.
3461          */
3462         if (host->caps & SDHCI_CAN_64BIT)
3463                 host->flags |= SDHCI_USE_64_BIT_DMA;
3464
3465         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3466                 ret = sdhci_set_dma_mask(host);
3467
3468                 if (!ret && host->ops->enable_dma)
3469                         ret = host->ops->enable_dma(host);
3470
3471                 if (ret) {
3472                         pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3473                                 mmc_hostname(mmc));
3474                         host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3475
3476                         ret = 0;
3477                 }
3478         }
3479
3480         /* SDMA does not support 64-bit DMA */
3481         if (host->flags & SDHCI_USE_64_BIT_DMA)
3482                 host->flags &= ~SDHCI_USE_SDMA;
3483
3484         if (host->flags & SDHCI_USE_ADMA) {
3485                 dma_addr_t dma;
3486                 void *buf;
3487
3488                 /*
3489                  * The DMA descriptor table size is calculated as the maximum
3490                  * number of segments times 2, to allow for an alignment
3491                  * descriptor for each segment, plus 1 for a nop end descriptor,
3492                  * all multipled by the descriptor size.
3493                  */
3494                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
3495                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3496                                               SDHCI_ADMA2_64_DESC_SZ;
3497                         host->desc_sz = SDHCI_ADMA2_64_DESC_SZ;
3498                 } else {
3499                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3500                                               SDHCI_ADMA2_32_DESC_SZ;
3501                         host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3502                 }
3503
3504                 host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3505                 buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz +
3506                                          host->adma_table_sz, &dma, GFP_KERNEL);
3507                 if (!buf) {
3508                         pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3509                                 mmc_hostname(mmc));
3510                         host->flags &= ~SDHCI_USE_ADMA;
3511                 } else if ((dma + host->align_buffer_sz) &
3512                            (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3513                         pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3514                                 mmc_hostname(mmc));
3515                         host->flags &= ~SDHCI_USE_ADMA;
3516                         dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3517                                           host->adma_table_sz, buf, dma);
3518                 } else {
3519                         host->align_buffer = buf;
3520                         host->align_addr = dma;
3521
3522                         host->adma_table = buf + host->align_buffer_sz;
3523                         host->adma_addr = dma + host->align_buffer_sz;
3524                 }
3525         }
3526
3527         /*
3528          * If we use DMA, then it's up to the caller to set the DMA
3529          * mask, but PIO does not need the hw shim so we set a new
3530          * mask here in that case.
3531          */
3532         if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3533                 host->dma_mask = DMA_BIT_MASK(64);
3534                 mmc_dev(mmc)->dma_mask = &host->dma_mask;
3535         }
3536
3537         if (host->version >= SDHCI_SPEC_300)
3538                 host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3539                         >> SDHCI_CLOCK_BASE_SHIFT;
3540         else
3541                 host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3542                         >> SDHCI_CLOCK_BASE_SHIFT;
3543
3544         host->max_clk *= 1000000;
3545         if (host->max_clk == 0 || host->quirks &
3546                         SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3547                 if (!host->ops->get_max_clock) {
3548                         pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3549                                mmc_hostname(mmc));
3550                         ret = -ENODEV;
3551                         goto undma;
3552                 }
3553                 host->max_clk = host->ops->get_max_clock(host);
3554         }
3555
3556         /*
3557          * In case of Host Controller v3.00, find out whether clock
3558          * multiplier is supported.
3559          */
3560         host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3561                         SDHCI_CLOCK_MUL_SHIFT;
3562
3563         /*
3564          * In case the value in Clock Multiplier is 0, then programmable
3565          * clock mode is not supported, otherwise the actual clock
3566          * multiplier is one more than the value of Clock Multiplier
3567          * in the Capabilities Register.
3568          */
3569         if (host->clk_mul)
3570                 host->clk_mul += 1;
3571
3572         /*
3573          * Set host parameters.
3574          */
3575         max_clk = host->max_clk;
3576
3577         if (host->ops->get_min_clock)
3578                 mmc->f_min = host->ops->get_min_clock(host);
3579         else if (host->version >= SDHCI_SPEC_300) {
3580                 if (host->clk_mul) {
3581                         mmc->f_min = (host->max_clk * host->clk_mul) / 1024;
3582                         max_clk = host->max_clk * host->clk_mul;
3583                 } else
3584                         mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3585         } else
3586                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3587
3588         if (!mmc->f_max || mmc->f_max > max_clk)
3589                 mmc->f_max = max_clk;
3590
3591         if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3592                 host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3593                                         SDHCI_TIMEOUT_CLK_SHIFT;
3594
3595                 if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3596                         host->timeout_clk *= 1000;
3597
3598                 if (host->timeout_clk == 0) {
3599                         if (!host->ops->get_timeout_clock) {
3600                                 pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3601                                         mmc_hostname(mmc));
3602                                 ret = -ENODEV;
3603                                 goto undma;
3604                         }
3605
3606                         host->timeout_clk =
3607                                 DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3608                                              1000);
3609                 }
3610
3611                 if (override_timeout_clk)
3612                         host->timeout_clk = override_timeout_clk;
3613
3614                 mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3615                         host->ops->get_max_timeout_count(host) : 1 << 27;
3616                 mmc->max_busy_timeout /= host->timeout_clk;
3617         }
3618
3619         mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3620         mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3621
3622         if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3623                 host->flags |= SDHCI_AUTO_CMD12;
3624
3625         /* Auto-CMD23 stuff only works in ADMA or PIO. */
3626         if ((host->version >= SDHCI_SPEC_300) &&
3627             ((host->flags & SDHCI_USE_ADMA) ||
3628              !(host->flags & SDHCI_USE_SDMA)) &&
3629              !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3630                 host->flags |= SDHCI_AUTO_CMD23;
3631                 DBG("Auto-CMD23 available\n");
3632         } else {
3633                 DBG("Auto-CMD23 unavailable\n");
3634         }
3635
3636         /*
3637          * A controller may support 8-bit width, but the board itself
3638          * might not have the pins brought out.  Boards that support
3639          * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3640          * their platform code before calling sdhci_add_host(), and we
3641          * won't assume 8-bit width for hosts without that CAP.
3642          */
3643         if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3644                 mmc->caps |= MMC_CAP_4_BIT_DATA;
3645
3646         if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3647                 mmc->caps &= ~MMC_CAP_CMD23;
3648
3649         if (host->caps & SDHCI_CAN_DO_HISPD)
3650                 mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3651
3652         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3653             mmc_card_is_removable(mmc) &&
3654             mmc_gpio_get_cd(host->mmc) < 0)
3655                 mmc->caps |= MMC_CAP_NEEDS_POLL;
3656
3657         /* If vqmmc regulator and no 1.8V signalling, then there's no UHS */
3658         if (!IS_ERR(mmc->supply.vqmmc)) {
3659                 ret = regulator_enable(mmc->supply.vqmmc);
3660                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3661                                                     1950000))
3662                         host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3663                                          SDHCI_SUPPORT_SDR50 |
3664                                          SDHCI_SUPPORT_DDR50);
3665                 if (ret) {
3666                         pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3667                                 mmc_hostname(mmc), ret);
3668                         mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3669                 }
3670         }
3671
3672         if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3673                 host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3674                                  SDHCI_SUPPORT_DDR50);
3675         }
3676
3677         /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3678         if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3679                            SDHCI_SUPPORT_DDR50))
3680                 mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3681
3682         /* SDR104 supports also implies SDR50 support */
3683         if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3684                 mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3685                 /* SD3.0: SDR104 is supported so (for eMMC) the caps2
3686                  * field can be promoted to support HS200.
3687                  */
3688                 if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3689                         mmc->caps2 |= MMC_CAP2_HS200;
3690         } else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3691                 mmc->caps |= MMC_CAP_UHS_SDR50;
3692         }
3693
3694         if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3695             (host->caps1 & SDHCI_SUPPORT_HS400))
3696                 mmc->caps2 |= MMC_CAP2_HS400;
3697
3698         if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3699             (IS_ERR(mmc->supply.vqmmc) ||
3700              !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3701                                              1300000)))
3702                 mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3703
3704         if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3705             !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3706                 mmc->caps |= MMC_CAP_UHS_DDR50;
3707
3708         /* Does the host need tuning for SDR50? */
3709         if (host->caps1 & SDHCI_USE_SDR50_TUNING)
3710                 host->flags |= SDHCI_SDR50_NEEDS_TUNING;
3711
3712         /* Driver Type(s) (A, C, D) supported by the host */
3713         if (host->caps1 & SDHCI_DRIVER_TYPE_A)
3714                 mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
3715         if (host->caps1 & SDHCI_DRIVER_TYPE_C)
3716                 mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
3717         if (host->caps1 & SDHCI_DRIVER_TYPE_D)
3718                 mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
3719
3720         /* Initial value for re-tuning timer count */
3721         host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
3722                              SDHCI_RETUNING_TIMER_COUNT_SHIFT;
3723
3724         /*
3725          * In case Re-tuning Timer is not disabled, the actual value of
3726          * re-tuning timer will be 2 ^ (n - 1).
3727          */
3728         if (host->tuning_count)
3729                 host->tuning_count = 1 << (host->tuning_count - 1);
3730
3731         /* Re-tuning mode supported by the Host Controller */
3732         host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
3733                              SDHCI_RETUNING_MODE_SHIFT;
3734
3735         ocr_avail = 0;
3736
3737         /*
3738          * According to SD Host Controller spec v3.00, if the Host System
3739          * can afford more than 150mA, Host Driver should set XPC to 1. Also
3740          * the value is meaningful only if Voltage Support in the Capabilities
3741          * register is set. The actual current value is 4 times the register
3742          * value.
3743          */
3744         max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
3745         if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
3746                 int curr = regulator_get_current_limit(mmc->supply.vmmc);
3747                 if (curr > 0) {
3748
3749                         /* convert to SDHCI_MAX_CURRENT format */
3750                         curr = curr/1000;  /* convert to mA */
3751                         curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
3752
3753                         curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
3754                         max_current_caps =
3755                                 (curr << SDHCI_MAX_CURRENT_330_SHIFT) |
3756                                 (curr << SDHCI_MAX_CURRENT_300_SHIFT) |
3757                                 (curr << SDHCI_MAX_CURRENT_180_SHIFT);
3758                 }
3759         }
3760
3761         if (host->caps & SDHCI_CAN_VDD_330) {
3762                 ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
3763
3764                 mmc->max_current_330 = ((max_current_caps &
3765                                    SDHCI_MAX_CURRENT_330_MASK) >>
3766                                    SDHCI_MAX_CURRENT_330_SHIFT) *
3767                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3768         }
3769         if (host->caps & SDHCI_CAN_VDD_300) {
3770                 ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
3771
3772                 mmc->max_current_300 = ((max_current_caps &
3773                                    SDHCI_MAX_CURRENT_300_MASK) >>
3774                                    SDHCI_MAX_CURRENT_300_SHIFT) *
3775                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3776         }
3777         if (host->caps & SDHCI_CAN_VDD_180) {
3778                 ocr_avail |= MMC_VDD_165_195;
3779
3780                 mmc->max_current_180 = ((max_current_caps &
3781                                    SDHCI_MAX_CURRENT_180_MASK) >>
3782                                    SDHCI_MAX_CURRENT_180_SHIFT) *
3783                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3784         }
3785
3786         /* If OCR set by host, use it instead. */
3787         if (host->ocr_mask)
3788                 ocr_avail = host->ocr_mask;
3789
3790         /* If OCR set by external regulators, give it highest prio. */
3791         if (mmc->ocr_avail)
3792                 ocr_avail = mmc->ocr_avail;
3793
3794         mmc->ocr_avail = ocr_avail;
3795         mmc->ocr_avail_sdio = ocr_avail;
3796         if (host->ocr_avail_sdio)
3797                 mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
3798         mmc->ocr_avail_sd = ocr_avail;
3799         if (host->ocr_avail_sd)
3800                 mmc->ocr_avail_sd &= host->ocr_avail_sd;
3801         else /* normal SD controllers don't support 1.8V */
3802                 mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
3803         mmc->ocr_avail_mmc = ocr_avail;
3804         if (host->ocr_avail_mmc)
3805                 mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
3806
3807         if (mmc->ocr_avail == 0) {
3808                 pr_err("%s: Hardware doesn't report any support voltages.\n",
3809                        mmc_hostname(mmc));
3810                 ret = -ENODEV;
3811                 goto unreg;
3812         }
3813
3814         if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
3815                           MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
3816                           MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
3817             (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
3818                 host->flags |= SDHCI_SIGNALING_180;
3819
3820         if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
3821                 host->flags |= SDHCI_SIGNALING_120;
3822
3823         spin_lock_init(&host->lock);
3824
3825         /*
3826          * Maximum number of sectors in one transfer. Limited by SDMA boundary
3827          * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
3828          * is less anyway.
3829          */
3830         mmc->max_req_size = 524288;
3831
3832         /*
3833          * Maximum number of segments. Depends on if the hardware
3834          * can do scatter/gather or not.
3835          */
3836         if (host->flags & SDHCI_USE_ADMA) {
3837                 mmc->max_segs = SDHCI_MAX_SEGS;
3838         } else if (host->flags & SDHCI_USE_SDMA) {
3839                 mmc->max_segs = 1;
3840                 if (swiotlb_max_segment()) {
3841                         unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
3842                                                 IO_TLB_SEGSIZE;
3843                         mmc->max_req_size = min(mmc->max_req_size,
3844                                                 max_req_size);
3845                 }
3846         } else { /* PIO */
3847                 mmc->max_segs = SDHCI_MAX_SEGS;
3848         }
3849
3850         /*
3851          * Maximum segment size. Could be one segment with the maximum number
3852          * of bytes. When doing hardware scatter/gather, each entry cannot
3853          * be larger than 64 KiB though.
3854          */
3855         if (host->flags & SDHCI_USE_ADMA) {
3856                 if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
3857                         mmc->max_seg_size = 65535;
3858                 else
3859                         mmc->max_seg_size = 65536;
3860         } else {
3861                 mmc->max_seg_size = mmc->max_req_size;
3862         }
3863
3864         /*
3865          * Maximum block size. This varies from controller to controller and
3866          * is specified in the capabilities register.
3867          */
3868         if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
3869                 mmc->max_blk_size = 2;
3870         } else {
3871                 mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
3872                                 SDHCI_MAX_BLOCK_SHIFT;
3873                 if (mmc->max_blk_size >= 3) {
3874                         pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
3875                                 mmc_hostname(mmc));
3876                         mmc->max_blk_size = 0;
3877                 }
3878         }
3879
3880         mmc->max_blk_size = 512 << mmc->max_blk_size;
3881
3882         /*
3883          * Maximum block count.
3884          */
3885         mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
3886
3887         if (mmc->max_segs == 1) {
3888                 /* This may alter mmc->*_blk_* parameters */
3889                 ret = sdhci_allocate_bounce_buffer(host);
3890                 if (ret)
3891                         return ret;
3892         }
3893
3894         return 0;
3895
3896 unreg:
3897         if (!IS_ERR(mmc->supply.vqmmc))
3898                 regulator_disable(mmc->supply.vqmmc);
3899 undma:
3900         if (host->align_buffer)
3901                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3902                                   host->adma_table_sz, host->align_buffer,
3903                                   host->align_addr);
3904         host->adma_table = NULL;
3905         host->align_buffer = NULL;
3906
3907         return ret;
3908 }
3909 EXPORT_SYMBOL_GPL(sdhci_setup_host);
3910
3911 void sdhci_cleanup_host(struct sdhci_host *host)
3912 {
3913         struct mmc_host *mmc = host->mmc;
3914
3915         if (!IS_ERR(mmc->supply.vqmmc))
3916                 regulator_disable(mmc->supply.vqmmc);
3917
3918         if (host->align_buffer)
3919                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3920                                   host->adma_table_sz, host->align_buffer,
3921                                   host->align_addr);
3922         host->adma_table = NULL;
3923         host->align_buffer = NULL;
3924 }
3925 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
3926
3927 int __sdhci_add_host(struct sdhci_host *host)
3928 {
3929         struct mmc_host *mmc = host->mmc;
3930         int ret;
3931
3932         /*
3933          * Init tasklets.
3934          */
3935         tasklet_init(&host->finish_tasklet,
3936                 sdhci_tasklet_finish, (unsigned long)host);
3937
3938         timer_setup(&host->timer, sdhci_timeout_timer, 0);
3939         timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);
3940
3941         init_waitqueue_head(&host->buf_ready_int);
3942
3943         sdhci_init(host, 0);
3944
3945         ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
3946                                    IRQF_SHARED, mmc_hostname(mmc), host);
3947         if (ret) {
3948                 pr_err("%s: Failed to request IRQ %d: %d\n",
3949                        mmc_hostname(mmc), host->irq, ret);
3950                 goto untasklet;
3951         }
3952
3953         ret = sdhci_led_register(host);
3954         if (ret) {
3955                 pr_err("%s: Failed to register LED device: %d\n",
3956                        mmc_hostname(mmc), ret);
3957                 goto unirq;
3958         }
3959
3960         mmiowb();
3961
3962         ret = mmc_add_host(mmc);
3963         if (ret)
3964                 goto unled;
3965
3966         pr_info("%s: SDHCI controller on %s [%s] using %s\n",
3967                 mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
3968                 (host->flags & SDHCI_USE_ADMA) ?
3969                 (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
3970                 (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
3971
3972         sdhci_enable_card_detection(host);
3973
3974         return 0;
3975
3976 unled:
3977         sdhci_led_unregister(host);
3978 unirq:
3979         sdhci_do_reset(host, SDHCI_RESET_ALL);
3980         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3981         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3982         free_irq(host->irq, host);
3983 untasklet:
3984         tasklet_kill(&host->finish_tasklet);
3985
3986         return ret;
3987 }
3988 EXPORT_SYMBOL_GPL(__sdhci_add_host);
3989
3990 int sdhci_add_host(struct sdhci_host *host)
3991 {
3992         int ret;
3993
3994         ret = sdhci_setup_host(host);
3995         if (ret)
3996                 return ret;
3997
3998         ret = __sdhci_add_host(host);
3999         if (ret)
4000                 goto cleanup;
4001
4002         return 0;
4003
4004 cleanup:
4005         sdhci_cleanup_host(host);
4006
4007         return ret;
4008 }
4009 EXPORT_SYMBOL_GPL(sdhci_add_host);
4010
4011 void sdhci_remove_host(struct sdhci_host *host, int dead)
4012 {
4013         struct mmc_host *mmc = host->mmc;
4014         unsigned long flags;
4015
4016         if (dead) {
4017                 spin_lock_irqsave(&host->lock, flags);
4018
4019                 host->flags |= SDHCI_DEVICE_DEAD;
4020
4021                 if (sdhci_has_requests(host)) {
4022                         pr_err("%s: Controller removed during "
4023                                 " transfer!\n", mmc_hostname(mmc));
4024                         sdhci_error_out_mrqs(host, -ENOMEDIUM);
4025                 }
4026
4027                 spin_unlock_irqrestore(&host->lock, flags);
4028         }
4029
4030         sdhci_disable_card_detection(host);
4031
4032         mmc_remove_host(mmc);
4033
4034         sdhci_led_unregister(host);
4035
4036         if (!dead)
4037                 sdhci_do_reset(host, SDHCI_RESET_ALL);
4038
4039         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4040         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4041         free_irq(host->irq, host);
4042
4043         del_timer_sync(&host->timer);
4044         del_timer_sync(&host->data_timer);
4045
4046         tasklet_kill(&host->finish_tasklet);
4047
4048         if (!IS_ERR(mmc->supply.vqmmc))
4049                 regulator_disable(mmc->supply.vqmmc);
4050
4051         if (host->align_buffer)
4052                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4053                                   host->adma_table_sz, host->align_buffer,
4054                                   host->align_addr);
4055
4056         host->adma_table = NULL;
4057         host->align_buffer = NULL;
4058 }
4059
4060 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4061
4062 void sdhci_free_host(struct sdhci_host *host)
4063 {
4064         mmc_free_host(host->mmc);
4065 }
4066
4067 EXPORT_SYMBOL_GPL(sdhci_free_host);
4068
4069 /*****************************************************************************\
4070  *                                                                           *
4071  * Driver init/exit                                                          *
4072  *                                                                           *
4073 \*****************************************************************************/
4074
4075 static int __init sdhci_drv_init(void)
4076 {
4077         pr_info(DRIVER_NAME
4078                 ": Secure Digital Host Controller Interface driver\n");
4079         pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4080
4081         return 0;
4082 }
4083
4084 static void __exit sdhci_drv_exit(void)
4085 {
4086 }
4087
4088 module_init(sdhci_drv_init);
4089 module_exit(sdhci_drv_exit);
4090
4091 module_param(debug_quirks, uint, 0444);
4092 module_param(debug_quirks2, uint, 0444);
4093
4094 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4095 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4096 MODULE_LICENSE("GPL");
4097
4098 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4099 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");