drivers/mmc/host/sdhci.c

   1 /*
   2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   3  *
   4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or (at
   9  * your option) any later version.
  10  *
  11  * Thanks to the following companies for their support:
  12  *
  13  *     - JMicron (hardware and technical support)
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/ktime.h>
  18 #include <linux/highmem.h>
  19 #include <linux/io.h>
  20 #include <linux/module.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sizes.h>
  25 #include <linux/swiotlb.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/of.h>
  29
  30 #include <linux/leds.h>
  31
  32 #include <linux/mmc/mmc.h>
  33 #include <linux/mmc/host.h>
  34 #include <linux/mmc/card.h>
  35 #include <linux/mmc/sdio.h>
  36 #include <linux/mmc/slot-gpio.h>
  37
  38 #include "sdhci.h"
  39
  40 #define DRIVER_NAME "sdhci"
  41
  42 #define DBG(f, x...) \
  43         pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45 #define SDHCI_DUMP(f, x...) \
  46         pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  47
  48 #define MAX_TUNING_LOOP 40
  49
  50 static unsigned int debug_quirks = 0;
  51 static unsigned int debug_quirks2;
  52
  53 static void sdhci_finish_data(struct sdhci_host *);
  54
  55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  56
  57 void sdhci_dumpregs(struct sdhci_host *host)
  58 {
  59         SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  60
  61         SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  62                    sdhci_readl(host, SDHCI_DMA_ADDRESS),
  63                    sdhci_readw(host, SDHCI_HOST_VERSION));
  64         SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  65                    sdhci_readw(host, SDHCI_BLOCK_SIZE),
  66                    sdhci_readw(host, SDHCI_BLOCK_COUNT));
  67         SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  68                    sdhci_readl(host, SDHCI_ARGUMENT),
  69                    sdhci_readw(host, SDHCI_TRANSFER_MODE));
  70         SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  71                    sdhci_readl(host, SDHCI_PRESENT_STATE),
  72                    sdhci_readb(host, SDHCI_HOST_CONTROL));
  73         SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  74                    sdhci_readb(host, SDHCI_POWER_CONTROL),
  75                    sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  76         SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  77                    sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  78                    sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  79         SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  80                    sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  81                    sdhci_readl(host, SDHCI_INT_STATUS));
  82         SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  83                    sdhci_readl(host, SDHCI_INT_ENABLE),
  84                    sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  85         SDHCI_DUMP("AC12 err:  0x%08x | Slot int: 0x%08x\n",
  86                    sdhci_readw(host, SDHCI_ACMD12_ERR),
  87                    sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  88         SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  89                    sdhci_readl(host, SDHCI_CAPABILITIES),
  90                    sdhci_readl(host, SDHCI_CAPABILITIES_1));
  91         SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  92                    sdhci_readw(host, SDHCI_COMMAND),
  93                    sdhci_readl(host, SDHCI_MAX_CURRENT));
  94         SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  95                    sdhci_readl(host, SDHCI_RESPONSE),
  96                    sdhci_readl(host, SDHCI_RESPONSE + 4));
  97         SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  98                    sdhci_readl(host, SDHCI_RESPONSE + 8),
  99                    sdhci_readl(host, SDHCI_RESPONSE + 12));
 100         SDHCI_DUMP("Host ctl2: 0x%08x\n",
 101                    sdhci_readw(host, SDHCI_HOST_CONTROL2));
 102
 103         if (host->flags & SDHCI_USE_ADMA) {
 104                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
 105                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 106                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 107                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 108                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 109                 } else {
 110                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 111                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 112                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 113                 }
 114         }
 115
 116         SDHCI_DUMP("============================================\n");
 117 }
 118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120 /*****************************************************************************\
 121  *                                                                           *
 122  * Low level functions                                                       *
 123  *                                                                           *
 124 \*****************************************************************************/
 125
 126 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 127 {
 128         return cmd->data || cmd->flags & MMC_RSP_BUSY;
 129 }
 130
 131 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 132 {
 133         u32 present;
 134
 135         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 136             !mmc_card_is_removable(host->mmc))
 137                 return;
 138
 139         if (enable) {
 140                 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 141                                       SDHCI_CARD_PRESENT;
 142
 143                 host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 144                                        SDHCI_INT_CARD_INSERT;
 145         } else {
 146                 host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 147         }
 148
 149         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 150         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 151 }
 152
 153 static void sdhci_enable_card_detection(struct sdhci_host *host)
 154 {
 155         sdhci_set_card_detection(host, true);
 156 }
 157
 158 static void sdhci_disable_card_detection(struct sdhci_host *host)
 159 {
 160         sdhci_set_card_detection(host, false);
 161 }
 162
 163 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 164 {
 165         if (host->bus_on)
 166                 return;
 167         host->bus_on = true;
 168         pm_runtime_get_noresume(host->mmc->parent);
 169 }
 170
 171 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 172 {
 173         if (!host->bus_on)
 174                 return;
 175         host->bus_on = false;
 176         pm_runtime_put_noidle(host->mmc->parent);
 177 }
 178
 179 void sdhci_reset(struct sdhci_host *host, u8 mask)
 180 {
 181         ktime_t timeout;
 182
 183         sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 184
 185         if (mask & SDHCI_RESET_ALL) {
 186                 host->clock = 0;
 187                 /* Reset-all turns off SD Bus Power */
 188                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 189                         sdhci_runtime_pm_bus_off(host);
 190         }
 191
 192         /* Wait max 100 ms */
 193         timeout = ktime_add_ms(ktime_get(), 100);
 194
 195         /* hw clears the bit when it's done */
 196         while (sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask) {
 197                 if (ktime_after(ktime_get(), timeout)) {
 198                         pr_err("%s: Reset 0x%x never completed.\n",
 199                                 mmc_hostname(host->mmc), (int)mask);
 200                         sdhci_dumpregs(host);
 201                         return;
 202                 }
 203                 udelay(10);
 204         }
 205 }
 206 EXPORT_SYMBOL_GPL(sdhci_reset);
 207
 208 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 209 {
 210         if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 211                 struct mmc_host *mmc = host->mmc;
 212
 213                 if (!mmc->ops->get_cd(mmc))
 214                         return;
 215         }
 216
 217         host->ops->reset(host, mask);
 218
 219         if (mask & SDHCI_RESET_ALL) {
 220                 if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 221                         if (host->ops->enable_dma)
 222                                 host->ops->enable_dma(host);
 223                 }
 224
 225                 /* Resetting the controller clears many */
 226                 host->preset_enabled = false;
 227         }
 228 }
 229
 230 static void sdhci_set_default_irqs(struct sdhci_host *host)
 231 {
 232         host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 233                     SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 234                     SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 235                     SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 236                     SDHCI_INT_RESPONSE;
 237
 238         if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 239             host->tuning_mode == SDHCI_TUNING_MODE_3)
 240                 host->ier |= SDHCI_INT_RETUNE;
 241
 242         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 243         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 244 }
 245
 246 static void sdhci_init(struct sdhci_host *host, int soft)
 247 {
 248         struct mmc_host *mmc = host->mmc;
 249
 250         if (soft)
 251                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 252         else
 253                 sdhci_do_reset(host, SDHCI_RESET_ALL);
 254
 255         sdhci_set_default_irqs(host);
 256
 257         host->cqe_on = false;
 258
 259         if (soft) {
 260                 /* force clock reconfiguration */
 261                 host->clock = 0;
 262                 mmc->ops->set_ios(mmc, &mmc->ios);
 263         }
 264 }
 265
 266 static void sdhci_reinit(struct sdhci_host *host)
 267 {
 268         sdhci_init(host, 0);
 269         sdhci_enable_card_detection(host);
 270 }
 271
 272 static void __sdhci_led_activate(struct sdhci_host *host)
 273 {
 274         u8 ctrl;
 275
 276         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 277         ctrl |= SDHCI_CTRL_LED;
 278         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 279 }
 280
 281 static void __sdhci_led_deactivate(struct sdhci_host *host)
 282 {
 283         u8 ctrl;
 284
 285         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 286         ctrl &= ~SDHCI_CTRL_LED;
 287         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 288 }
 289
 290 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
 291 static void sdhci_led_control(struct led_classdev *led,
 292                               enum led_brightness brightness)
 293 {
 294         struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 295         unsigned long flags;
 296
 297         spin_lock_irqsave(&host->lock, flags);
 298
 299         if (host->runtime_suspended)
 300                 goto out;
 301
 302         if (brightness == LED_OFF)
 303                 __sdhci_led_deactivate(host);
 304         else
 305                 __sdhci_led_activate(host);
 306 out:
 307         spin_unlock_irqrestore(&host->lock, flags);
 308 }
 309
 310 static int sdhci_led_register(struct sdhci_host *host)
 311 {
 312         struct mmc_host *mmc = host->mmc;
 313
 314         snprintf(host->led_name, sizeof(host->led_name),
 315                  "%s::", mmc_hostname(mmc));
 316
 317         host->led.name = host->led_name;
 318         host->led.brightness = LED_OFF;
 319         host->led.default_trigger = mmc_hostname(mmc);
 320         host->led.brightness_set = sdhci_led_control;
 321
 322         return led_classdev_register(mmc_dev(mmc), &host->led);
 323 }
 324
 325 static void sdhci_led_unregister(struct sdhci_host *host)
 326 {
 327         led_classdev_unregister(&host->led);
 328 }
 329
 330 static inline void sdhci_led_activate(struct sdhci_host *host)
 331 {
 332 }
 333
 334 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 335 {
 336 }
 337
 338 #else
 339
 340 static inline int sdhci_led_register(struct sdhci_host *host)
 341 {
 342         return 0;
 343 }
 344
 345 static inline void sdhci_led_unregister(struct sdhci_host *host)
 346 {
 347 }
 348
 349 static inline void sdhci_led_activate(struct sdhci_host *host)
 350 {
 351         __sdhci_led_activate(host);
 352 }
 353
 354 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 355 {
 356         __sdhci_led_deactivate(host);
 357 }
 358
 359 #endif
 360
 361 /*****************************************************************************\
 362  *                                                                           *
 363  * Core functions                                                            *
 364  *                                                                           *
 365 \*****************************************************************************/
 366
 367 static void sdhci_read_block_pio(struct sdhci_host *host)
 368 {
 369         unsigned long flags;
 370         size_t blksize, len, chunk;
 371         u32 uninitialized_var(scratch);
 372         u8 *buf;
 373
 374         DBG("PIO reading\n");
 375
 376         blksize = host->data->blksz;
 377         chunk = 0;
 378
 379         local_irq_save(flags);
 380
 381         while (blksize) {
 382                 BUG_ON(!sg_miter_next(&host->sg_miter));
 383
 384                 len = min(host->sg_miter.length, blksize);
 385
 386                 blksize -= len;
 387                 host->sg_miter.consumed = len;
 388
 389                 buf = host->sg_miter.addr;
 390
 391                 while (len) {
 392                         if (chunk == 0) {
 393                                 scratch = sdhci_readl(host, SDHCI_BUFFER);
 394                                 chunk = 4;
 395                         }
 396
 397                         *buf = scratch & 0xFF;
 398
 399                         buf++;
 400                         scratch >>= 8;
 401                         chunk--;
 402                         len--;
 403                 }
 404         }
 405
 406         sg_miter_stop(&host->sg_miter);
 407
 408         local_irq_restore(flags);
 409 }
 410
 411 static void sdhci_write_block_pio(struct sdhci_host *host)
 412 {
 413         unsigned long flags;
 414         size_t blksize, len, chunk;
 415         u32 scratch;
 416         u8 *buf;
 417
 418         DBG("PIO writing\n");
 419
 420         blksize = host->data->blksz;
 421         chunk = 0;
 422         scratch = 0;
 423
 424         local_irq_save(flags);
 425
 426         while (blksize) {
 427                 BUG_ON(!sg_miter_next(&host->sg_miter));
 428
 429                 len = min(host->sg_miter.length, blksize);
 430
 431                 blksize -= len;
 432                 host->sg_miter.consumed = len;
 433
 434                 buf = host->sg_miter.addr;
 435
 436                 while (len) {
 437                         scratch |= (u32)*buf << (chunk * 8);
 438
 439                         buf++;
 440                         chunk++;
 441                         len--;
 442
 443                         if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 444                                 sdhci_writel(host, scratch, SDHCI_BUFFER);
 445                                 chunk = 0;
 446                                 scratch = 0;
 447                         }
 448                 }
 449         }
 450
 451         sg_miter_stop(&host->sg_miter);
 452
 453         local_irq_restore(flags);
 454 }
 455
 456 static void sdhci_transfer_pio(struct sdhci_host *host)
 457 {
 458         u32 mask;
 459
 460         if (host->blocks == 0)
 461                 return;
 462
 463         if (host->data->flags & MMC_DATA_READ)
 464                 mask = SDHCI_DATA_AVAILABLE;
 465         else
 466                 mask = SDHCI_SPACE_AVAILABLE;
 467
 468         /*
 469          * Some controllers (JMicron JMB38x) mess up the buffer bits
 470          * for transfers < 4 bytes. As long as it is just one block,
 471          * we can ignore the bits.
 472          */
 473         if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 474                 (host->data->blocks == 1))
 475                 mask = ~0;
 476
 477         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 478                 if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 479                         udelay(100);
 480
 481                 if (host->data->flags & MMC_DATA_READ)
 482                         sdhci_read_block_pio(host);
 483                 else
 484                         sdhci_write_block_pio(host);
 485
 486                 host->blocks--;
 487                 if (host->blocks == 0)
 488                         break;
 489         }
 490
 491         DBG("PIO transfer complete.\n");
 492 }
 493
 494 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 495                                   struct mmc_data *data, int cookie)
 496 {
 497         int sg_count;
 498
 499         /*
 500          * If the data buffers are already mapped, return the previous
 501          * dma_map_sg() result.
 502          */
 503         if (data->host_cookie == COOKIE_PRE_MAPPED)
 504                 return data->sg_count;
 505
 506         /* Bounce write requests to the bounce buffer */
 507         if (host->bounce_buffer) {
 508                 unsigned int length = data->blksz * data->blocks;
 509
 510                 if (length > host->bounce_buffer_size) {
 511                         pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 512                                mmc_hostname(host->mmc), length,
 513                                host->bounce_buffer_size);
 514                         return -EIO;
 515                 }
 516                 if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 517                         /* Copy the data to the bounce buffer */
 518                         sg_copy_to_buffer(data->sg, data->sg_len,
 519                                           host->bounce_buffer,
 520                                           length);
 521                 }
 522                 /* Switch ownership to the DMA */
 523                 dma_sync_single_for_device(host->mmc->parent,
 524                                            host->bounce_addr,
 525                                            host->bounce_buffer_size,
 526                                            mmc_get_dma_dir(data));
 527                 /* Just a dummy value */
 528                 sg_count = 1;
 529         } else {
 530                 /* Just access the data directly from memory */
 531                 sg_count = dma_map_sg(mmc_dev(host->mmc),
 532                                       data->sg, data->sg_len,
 533                                       mmc_get_dma_dir(data));
 534         }
 535
 536         if (sg_count == 0)
 537                 return -ENOSPC;
 538
 539         data->sg_count = sg_count;
 540         data->host_cookie = cookie;
 541
 542         return sg_count;
 543 }
 544
 545 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 546 {
 547         local_irq_save(*flags);
 548         return kmap_atomic(sg_page(sg)) + sg->offset;
 549 }
 550
 551 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 552 {
 553         kunmap_atomic(buffer);
 554         local_irq_restore(*flags);
 555 }
 556
 557 static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
 558                                   dma_addr_t addr, int len, unsigned cmd)
 559 {
 560         struct sdhci_adma2_64_desc *dma_desc = desc;
 561
 562         /* 32-bit and 64-bit descriptors have these members in same position */
 563         dma_desc->cmd = cpu_to_le16(cmd);
 564         dma_desc->len = cpu_to_le16(len);
 565         dma_desc->addr_lo = cpu_to_le32((u32)addr);
 566
 567         if (host->flags & SDHCI_USE_64_BIT_DMA)
 568                 dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
 569 }
 570
 571 static void sdhci_adma_mark_end(void *desc)
 572 {
 573         struct sdhci_adma2_64_desc *dma_desc = desc;
 574
 575         /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 576         dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 577 }
 578
 579 static void sdhci_adma_table_pre(struct sdhci_host *host,
 580         struct mmc_data *data, int sg_count)
 581 {
 582         struct scatterlist *sg;
 583         unsigned long flags;
 584         dma_addr_t addr, align_addr;
 585         void *desc, *align;
 586         char *buffer;
 587         int len, offset, i;
 588
 589         /*
 590          * The spec does not specify endianness of descriptor table.
 591          * We currently guess that it is LE.
 592          */
 593
 594         host->sg_count = sg_count;
 595
 596         desc = host->adma_table;
 597         align = host->align_buffer;
 598
 599         align_addr = host->align_addr;
 600
 601         for_each_sg(data->sg, sg, host->sg_count, i) {
 602                 addr = sg_dma_address(sg);
 603                 len = sg_dma_len(sg);
 604
 605                 /*
 606                  * The SDHCI specification states that ADMA addresses must
 607                  * be 32-bit aligned. If they aren't, then we use a bounce
 608                  * buffer for the (up to three) bytes that screw up the
 609                  * alignment.
 610                  */
 611                 offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 612                          SDHCI_ADMA2_MASK;
 613                 if (offset) {
 614                         if (data->flags & MMC_DATA_WRITE) {
 615                                 buffer = sdhci_kmap_atomic(sg, &flags);
 616                                 memcpy(align, buffer, offset);
 617                                 sdhci_kunmap_atomic(buffer, &flags);
 618                         }
 619
 620                         /* tran, valid */
 621                         sdhci_adma_write_desc(host, desc, align_addr, offset,
 622                                               ADMA2_TRAN_VALID);
 623
 624                         BUG_ON(offset > 65536);
 625
 626                         align += SDHCI_ADMA2_ALIGN;
 627                         align_addr += SDHCI_ADMA2_ALIGN;
 628
 629                         desc += host->desc_sz;
 630
 631                         addr += offset;
 632                         len -= offset;
 633                 }
 634
 635                 BUG_ON(len > 65536);
 636
 637                 if (len) {
 638                         /* tran, valid */
 639                         sdhci_adma_write_desc(host, desc, addr, len,
 640                                               ADMA2_TRAN_VALID);
 641                         desc += host->desc_sz;
 642                 }
 643
 644                 /*
 645                  * If this triggers then we have a calculation bug
 646                  * somewhere. :/
 647                  */
 648                 WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 649         }
 650
 651         if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 652                 /* Mark the last descriptor as the terminating descriptor */
 653                 if (desc != host->adma_table) {
 654                         desc -= host->desc_sz;
 655                         sdhci_adma_mark_end(desc);
 656                 }
 657         } else {
 658                 /* Add a terminating entry - nop, end, valid */
 659                 sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
 660         }
 661 }
 662
 663 static void sdhci_adma_table_post(struct sdhci_host *host,
 664         struct mmc_data *data)
 665 {
 666         struct scatterlist *sg;
 667         int i, size;
 668         void *align;
 669         char *buffer;
 670         unsigned long flags;
 671
 672         if (data->flags & MMC_DATA_READ) {
 673                 bool has_unaligned = false;
 674
 675                 /* Do a quick scan of the SG list for any unaligned mappings */
 676                 for_each_sg(data->sg, sg, host->sg_count, i)
 677                         if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 678                                 has_unaligned = true;
 679                                 break;
 680                         }
 681
 682                 if (has_unaligned) {
 683                         dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 684                                             data->sg_len, DMA_FROM_DEVICE);
 685
 686                         align = host->align_buffer;
 687
 688                         for_each_sg(data->sg, sg, host->sg_count, i) {
 689                                 if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 690                                         size = SDHCI_ADMA2_ALIGN -
 691                                                (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 692
 693                                         buffer = sdhci_kmap_atomic(sg, &flags);
 694                                         memcpy(buffer, align, size);
 695                                         sdhci_kunmap_atomic(buffer, &flags);
 696
 697                                         align += SDHCI_ADMA2_ALIGN;
 698                                 }
 699                         }
 700                 }
 701         }
 702 }
 703
 704 static u32 sdhci_sdma_address(struct sdhci_host *host)
 705 {
 706         if (host->bounce_buffer)
 707                 return host->bounce_addr;
 708         else
 709                 return sg_dma_address(host->data->sg);
 710 }
 711
 712 static unsigned int sdhci_target_timeout(struct sdhci_host *host,
 713                                          struct mmc_command *cmd,
 714                                          struct mmc_data *data)
 715 {
 716         unsigned int target_timeout;
 717
 718         /* timeout in us */
 719         if (!data) {
 720                 target_timeout = cmd->busy_timeout * 1000;
 721         } else {
 722                 target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 723                 if (host->clock && data->timeout_clks) {
 724                         unsigned long long val;
 725
 726                         /*
 727                          * data->timeout_clks is in units of clock cycles.
 728                          * host->clock is in Hz.  target_timeout is in us.
 729                          * Hence, us = 1000000 * cycles / Hz.  Round up.
 730                          */
 731                         val = 1000000ULL * data->timeout_clks;
 732                         if (do_div(val, host->clock))
 733                                 target_timeout++;
 734                         target_timeout += val;
 735                 }
 736         }
 737
 738         return target_timeout;
 739 }
 740
 741 static void sdhci_calc_sw_timeout(struct sdhci_host *host,
 742                                   struct mmc_command *cmd)
 743 {
 744         struct mmc_data *data = cmd->data;
 745         struct mmc_host *mmc = host->mmc;
 746         struct mmc_ios *ios = &mmc->ios;
 747         unsigned char bus_width = 1 << ios->bus_width;
 748         unsigned int blksz;
 749         unsigned int freq;
 750         u64 target_timeout;
 751         u64 transfer_time;
 752
 753         target_timeout = sdhci_target_timeout(host, cmd, data);
 754         target_timeout *= NSEC_PER_USEC;
 755
 756         if (data) {
 757                 blksz = data->blksz;
 758                 freq = host->mmc->actual_clock ? : host->clock;
 759                 transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
 760                 do_div(transfer_time, freq);
 761                 /* multiply by '2' to account for any unknowns */
 762                 transfer_time = transfer_time * 2;
 763                 /* calculate timeout for the entire data */
 764                 host->data_timeout = data->blocks * target_timeout +
 765                                      transfer_time;
 766         } else {
 767                 host->data_timeout = target_timeout;
 768         }
 769
 770         if (host->data_timeout)
 771                 host->data_timeout += MMC_CMD_TRANSFER_TIME;
 772 }
 773
 774 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
 775                              bool *too_big)
 776 {
 777         u8 count;
 778         struct mmc_data *data = cmd->data;
 779         unsigned target_timeout, current_timeout;
 780
 781         *too_big = true;
 782
 783         /*
 784          * If the host controller provides us with an incorrect timeout
 785          * value, just skip the check and use 0xE.  The hardware may take
 786          * longer to time out, but that's much better than having a too-short
 787          * timeout value.
 788          */
 789         if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 790                 return 0xE;
 791
 792         /* Unspecified timeout, assume max */
 793         if (!data && !cmd->busy_timeout)
 794                 return 0xE;
 795
 796         /* timeout in us */
 797         target_timeout = sdhci_target_timeout(host, cmd, data);
 798
 799         /*
 800          * Figure out needed cycles.
 801          * We do this in steps in order to fit inside a 32 bit int.
 802          * The first step is the minimum timeout, which will have a
 803          * minimum resolution of 6 bits:
 804          * (1) 2^13*1000 > 2^22,
 805          * (2) host->timeout_clk < 2^16
 806          *     =>
 807          *     (1) / (2) > 2^6
 808          */
 809         count = 0;
 810         current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 811         while (current_timeout < target_timeout) {
 812                 count++;
 813                 current_timeout <<= 1;
 814                 if (count >= 0xF)
 815                         break;
 816         }
 817
 818         if (count >= 0xF) {
 819                 if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
 820                         DBG("Too large timeout 0x%x requested for CMD%d!\n",
 821                             count, cmd->opcode);
 822                 count = 0xE;
 823         } else {
 824                 *too_big = false;
 825         }
 826
 827         return count;
 828 }
 829
 830 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 831 {
 832         u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 833         u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 834
 835         if (host->flags & SDHCI_REQ_USE_DMA)
 836                 host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 837         else
 838                 host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 839
 840         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 841         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 842 }
 843
 844 static void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
 845 {
 846         if (enable)
 847                 host->ier |= SDHCI_INT_DATA_TIMEOUT;
 848         else
 849                 host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
 850         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 851         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 852 }
 853
 854 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 855 {
 856         u8 count;
 857
 858         if (host->ops->set_timeout) {
 859                 host->ops->set_timeout(host, cmd);
 860         } else {
 861                 bool too_big = false;
 862
 863                 count = sdhci_calc_timeout(host, cmd, &too_big);
 864
 865                 if (too_big &&
 866                     host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
 867                         sdhci_calc_sw_timeout(host, cmd);
 868                         sdhci_set_data_timeout_irq(host, false);
 869                 } else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
 870                         sdhci_set_data_timeout_irq(host, true);
 871                 }
 872
 873                 sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 874         }
 875 }
 876
 877 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 878 {
 879         u8 ctrl;
 880         struct mmc_data *data = cmd->data;
 881
 882         host->data_timeout = 0;
 883
 884         if (sdhci_data_line_cmd(cmd))
 885                 sdhci_set_timeout(host, cmd);
 886
 887         if (!data)
 888                 return;
 889
 890         WARN_ON(host->data);
 891
 892         /* Sanity checks */
 893         BUG_ON(data->blksz * data->blocks > 524288);
 894         BUG_ON(data->blksz > host->mmc->max_blk_size);
 895         BUG_ON(data->blocks > 65535);
 896
 897         host->data = data;
 898         host->data_early = 0;
 899         host->data->bytes_xfered = 0;
 900
 901         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 902                 struct scatterlist *sg;
 903                 unsigned int length_mask, offset_mask;
 904                 int i;
 905
 906                 host->flags |= SDHCI_REQ_USE_DMA;
 907
 908                 /*
 909                  * FIXME: This doesn't account for merging when mapping the
 910                  * scatterlist.
 911                  *
 912                  * The assumption here being that alignment and lengths are
 913                  * the same after DMA mapping to device address space.
 914                  */
 915                 length_mask = 0;
 916                 offset_mask = 0;
 917                 if (host->flags & SDHCI_USE_ADMA) {
 918                         if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
 919                                 length_mask = 3;
 920                                 /*
 921                                  * As we use up to 3 byte chunks to work
 922                                  * around alignment problems, we need to
 923                                  * check the offset as well.
 924                                  */
 925                                 offset_mask = 3;
 926                         }
 927                 } else {
 928                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
 929                                 length_mask = 3;
 930                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
 931                                 offset_mask = 3;
 932                 }
 933
 934                 if (unlikely(length_mask | offset_mask)) {
 935                         for_each_sg(data->sg, sg, data->sg_len, i) {
 936                                 if (sg->length & length_mask) {
 937                                         DBG("Reverting to PIO because of transfer size (%d)\n",
 938                                             sg->length);
 939                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 940                                         break;
 941                                 }
 942                                 if (sg->offset & offset_mask) {
 943                                         DBG("Reverting to PIO because of bad alignment\n");
 944                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 945                                         break;
 946                                 }
 947                         }
 948                 }
 949         }
 950
 951         if (host->flags & SDHCI_REQ_USE_DMA) {
 952                 int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
 953
 954                 if (sg_cnt <= 0) {
 955                         /*
 956                          * This only happens when someone fed
 957                          * us an invalid request.
 958                          */
 959                         WARN_ON(1);
 960                         host->flags &= ~SDHCI_REQ_USE_DMA;
 961                 } else if (host->flags & SDHCI_USE_ADMA) {
 962                         sdhci_adma_table_pre(host, data, sg_cnt);
 963
 964                         sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
 965                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 966                                 sdhci_writel(host,
 967                                              (u64)host->adma_addr >> 32,
 968                                              SDHCI_ADMA_ADDRESS_HI);
 969                 } else {
 970                         WARN_ON(sg_cnt != 1);
 971                         sdhci_writel(host, sdhci_sdma_address(host),
 972                                      SDHCI_DMA_ADDRESS);
 973                 }
 974         }
 975
 976         /*
 977          * Always adjust the DMA selection as some controllers
 978          * (e.g. JMicron) can't do PIO properly when the selection
 979          * is ADMA.
 980          */
 981         if (host->version >= SDHCI_SPEC_200) {
 982                 ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 983                 ctrl &= ~SDHCI_CTRL_DMA_MASK;
 984                 if ((host->flags & SDHCI_REQ_USE_DMA) &&
 985                         (host->flags & SDHCI_USE_ADMA)) {
 986                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 987                                 ctrl |= SDHCI_CTRL_ADMA64;
 988                         else
 989                                 ctrl |= SDHCI_CTRL_ADMA32;
 990                 } else {
 991                         ctrl |= SDHCI_CTRL_SDMA;
 992                 }
 993                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 994         }
 995
 996         if (!(host->flags & SDHCI_REQ_USE_DMA)) {
 997                 int flags;
 998
 999                 flags = SG_MITER_ATOMIC;
1000                 if (host->data->flags & MMC_DATA_READ)
1001                         flags |= SG_MITER_TO_SG;
1002                 else
1003                         flags |= SG_MITER_FROM_SG;
1004                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1005                 host->blocks = data->blocks;
1006         }
1007
1008         sdhci_set_transfer_irqs(host);
1009
1010         /* Set the DMA boundary value and block size */
1011         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
1012                      SDHCI_BLOCK_SIZE);
1013         sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
1014 }
1015
1016 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
1017                                     struct mmc_request *mrq)
1018 {
1019         return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
1020                !mrq->cap_cmd_during_tfr;
1021 }
1022
1023 static void sdhci_set_transfer_mode(struct sdhci_host *host,
1024         struct mmc_command *cmd)
1025 {
1026         u16 mode = 0;
1027         struct mmc_data *data = cmd->data;
1028
1029         if (data == NULL) {
1030                 if (host->quirks2 &
1031                         SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
1032                         sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1033                 } else {
1034                 /* clear Auto CMD settings for no data CMDs */
1035                         mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
1036                         sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
1037                                 SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
1038                 }
1039                 return;
1040         }
1041
1042         WARN_ON(!host->data);
1043
1044         if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
1045                 mode = SDHCI_TRNS_BLK_CNT_EN;
1046
1047         if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
1048                 mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
1049                 /*
1050                  * If we are sending CMD23, CMD12 never gets sent
1051                  * on successful completion (so no Auto-CMD12).
1052                  */
1053                 if (sdhci_auto_cmd12(host, cmd->mrq) &&
1054                     (cmd->opcode != SD_IO_RW_EXTENDED))
1055                         mode |= SDHCI_TRNS_AUTO_CMD12;
1056                 else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
1057                         mode |= SDHCI_TRNS_AUTO_CMD23;
1058                         sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
1059                 }
1060         }
1061
1062         if (data->flags & MMC_DATA_READ)
1063                 mode |= SDHCI_TRNS_READ;
1064         if (host->flags & SDHCI_REQ_USE_DMA)
1065                 mode |= SDHCI_TRNS_DMA;
1066
1067         sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1068 }
1069
1070 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1071 {
1072         return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1073                 ((mrq->cmd && mrq->cmd->error) ||
1074                  (mrq->sbc && mrq->sbc->error) ||
1075                  (mrq->data && ((mrq->data->error && !mrq->data->stop) ||
1076                                 (mrq->data->stop && mrq->data->stop->error))) ||
1077                  (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1078 }
1079
1080 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1081 {
1082         int i;
1083
1084         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1085                 if (host->mrqs_done[i] == mrq) {
1086                         WARN_ON(1);
1087                         return;
1088                 }
1089         }
1090
1091         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1092                 if (!host->mrqs_done[i]) {
1093                         host->mrqs_done[i] = mrq;
1094                         break;
1095                 }
1096         }
1097
1098         WARN_ON(i >= SDHCI_MAX_MRQS);
1099
1100         tasklet_schedule(&host->finish_tasklet);
1101 }
1102
1103 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1104 {
1105         if (host->cmd && host->cmd->mrq == mrq)
1106                 host->cmd = NULL;
1107
1108         if (host->data_cmd && host->data_cmd->mrq == mrq)
1109                 host->data_cmd = NULL;
1110
1111         if (host->data && host->data->mrq == mrq)
1112                 host->data = NULL;
1113
1114         if (sdhci_needs_reset(host, mrq))
1115                 host->pending_reset = true;
1116
1117         __sdhci_finish_mrq(host, mrq);
1118 }
1119
1120 static void sdhci_finish_data(struct sdhci_host *host)
1121 {
1122         struct mmc_command *data_cmd = host->data_cmd;
1123         struct mmc_data *data = host->data;
1124
1125         host->data = NULL;
1126         host->data_cmd = NULL;
1127
1128         if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1129             (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1130                 sdhci_adma_table_post(host, data);
1131
1132         /*
1133          * The specification states that the block count register must
1134          * be updated, but it does not specify at what point in the
1135          * data flow. That makes the register entirely useless to read
1136          * back so we have to assume that nothing made it to the card
1137          * in the event of an error.
1138          */
1139         if (data->error)
1140                 data->bytes_xfered = 0;
1141         else
1142                 data->bytes_xfered = data->blksz * data->blocks;
1143
1144         /*
1145          * Need to send CMD12 if -
1146          * a) open-ended multiblock transfer (no CMD23)
1147          * b) error in multiblock transfer
1148          */
1149         if (data->stop &&
1150             (data->error ||
1151              !data->mrq->sbc)) {
1152
1153                 /*
1154                  * The controller needs a reset of internal state machines
1155                  * upon error conditions.
1156                  */
1157                 if (data->error) {
1158                         if (!host->cmd || host->cmd == data_cmd)
1159                                 sdhci_do_reset(host, SDHCI_RESET_CMD);
1160                         sdhci_do_reset(host, SDHCI_RESET_DATA);
1161                 }
1162
1163                 /*
1164                  * 'cap_cmd_during_tfr' request must not use the command line
1165                  * after mmc_command_done() has been called. It is upper layer's
1166                  * responsibility to send the stop command if required.
1167                  */
1168                 if (data->mrq->cap_cmd_during_tfr) {
1169                         sdhci_finish_mrq(host, data->mrq);
1170                 } else {
1171                         /* Avoid triggering warning in sdhci_send_command() */
1172                         host->cmd = NULL;
1173                         sdhci_send_command(host, data->stop);
1174                 }
1175         } else {
1176                 sdhci_finish_mrq(host, data->mrq);
1177         }
1178 }
1179
1180 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1181                             unsigned long timeout)
1182 {
1183         if (sdhci_data_line_cmd(mrq->cmd))
1184                 mod_timer(&host->data_timer, timeout);
1185         else
1186                 mod_timer(&host->timer, timeout);
1187 }
1188
1189 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1190 {
1191         if (sdhci_data_line_cmd(mrq->cmd))
1192                 del_timer(&host->data_timer);
1193         else
1194                 del_timer(&host->timer);
1195 }
1196
1197 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1198 {
1199         int flags;
1200         u32 mask;
1201         unsigned long timeout;
1202
1203         WARN_ON(host->cmd);
1204
1205         /* Initially, a command has no error */
1206         cmd->error = 0;
1207
1208         if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1209             cmd->opcode == MMC_STOP_TRANSMISSION)
1210                 cmd->flags |= MMC_RSP_BUSY;
1211
1212         /* Wait max 10 ms */
1213         timeout = 10;
1214
1215         mask = SDHCI_CMD_INHIBIT;
1216         if (sdhci_data_line_cmd(cmd))
1217                 mask |= SDHCI_DATA_INHIBIT;
1218
1219         /* We shouldn't wait for data inihibit for stop commands, even
1220            though they might use busy signaling */
1221         if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1222                 mask &= ~SDHCI_DATA_INHIBIT;
1223
1224         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1225                 if (timeout == 0) {
1226                         pr_err("%s: Controller never released inhibit bit(s).\n",
1227                                mmc_hostname(host->mmc));
1228                         sdhci_dumpregs(host);
1229                         cmd->error = -EIO;
1230                         sdhci_finish_mrq(host, cmd->mrq);
1231                         return;
1232                 }
1233                 timeout--;
1234                 mdelay(1);
1235         }
1236
1237         host->cmd = cmd;
1238         if (sdhci_data_line_cmd(cmd)) {
1239                 WARN_ON(host->data_cmd);
1240                 host->data_cmd = cmd;
1241         }
1242
1243         sdhci_prepare_data(host, cmd);
1244
1245         sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1246
1247         sdhci_set_transfer_mode(host, cmd);
1248
1249         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1250                 pr_err("%s: Unsupported response type!\n",
1251                         mmc_hostname(host->mmc));
1252                 cmd->error = -EINVAL;
1253                 sdhci_finish_mrq(host, cmd->mrq);
1254                 return;
1255         }
1256
1257         if (!(cmd->flags & MMC_RSP_PRESENT))
1258                 flags = SDHCI_CMD_RESP_NONE;
1259         else if (cmd->flags & MMC_RSP_136)
1260                 flags = SDHCI_CMD_RESP_LONG;
1261         else if (cmd->flags & MMC_RSP_BUSY)
1262                 flags = SDHCI_CMD_RESP_SHORT_BUSY;
1263         else
1264                 flags = SDHCI_CMD_RESP_SHORT;
1265
1266         if (cmd->flags & MMC_RSP_CRC)
1267                 flags |= SDHCI_CMD_CRC;
1268         if (cmd->flags & MMC_RSP_OPCODE)
1269                 flags |= SDHCI_CMD_INDEX;
1270
1271         /* CMD19 is special in that the Data Present Select should be set */
1272         if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1273             cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1274                 flags |= SDHCI_CMD_DATA;
1275
1276         timeout = jiffies;
1277         if (host->data_timeout)
1278                 timeout += nsecs_to_jiffies(host->data_timeout);
1279         else if (!cmd->data && cmd->busy_timeout > 9000)
1280                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1281         else
1282                 timeout += 10 * HZ;
1283         sdhci_mod_timer(host, cmd->mrq, timeout);
1284
1285         sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1286 }
1287 EXPORT_SYMBOL_GPL(sdhci_send_command);
1288
1289 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1290 {
1291         int i, reg;
1292
1293         for (i = 0; i < 4; i++) {
1294                 reg = SDHCI_RESPONSE + (3 - i) * 4;
1295                 cmd->resp[i] = sdhci_readl(host, reg);
1296         }
1297
1298         if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1299                 return;
1300
1301         /* CRC is stripped so we need to do some shifting */
1302         for (i = 0; i < 4; i++) {
1303                 cmd->resp[i] <<= 8;
1304                 if (i != 3)
1305                         cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1306         }
1307 }
1308
1309 static void sdhci_finish_command(struct sdhci_host *host)
1310 {
1311         struct mmc_command *cmd = host->cmd;
1312
1313         host->cmd = NULL;
1314
1315         if (cmd->flags & MMC_RSP_PRESENT) {
1316                 if (cmd->flags & MMC_RSP_136) {
1317                         sdhci_read_rsp_136(host, cmd);
1318                 } else {
1319                         cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1320                 }
1321         }
1322
1323         if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1324                 mmc_command_done(host->mmc, cmd->mrq);
1325
1326         /*
1327          * The host can send and interrupt when the busy state has
1328          * ended, allowing us to wait without wasting CPU cycles.
1329          * The busy signal uses DAT0 so this is similar to waiting
1330          * for data to complete.
1331          *
1332          * Note: The 1.0 specification is a bit ambiguous about this
1333          *       feature so there might be some problems with older
1334          *       controllers.
1335          */
1336         if (cmd->flags & MMC_RSP_BUSY) {
1337                 if (cmd->data) {
1338                         DBG("Cannot wait for busy signal when also doing a data transfer");
1339                 } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1340                            cmd == host->data_cmd) {
1341                         /* Command complete before busy is ended */
1342                         return;
1343                 }
1344         }
1345
1346         /* Finished CMD23, now send actual command. */
1347         if (cmd == cmd->mrq->sbc) {
1348                 sdhci_send_command(host, cmd->mrq->cmd);
1349         } else {
1350
1351                 /* Processed actual command. */
1352                 if (host->data && host->data_early)
1353                         sdhci_finish_data(host);
1354
1355                 if (!cmd->data)
1356                         sdhci_finish_mrq(host, cmd->mrq);
1357         }
1358 }
1359
1360 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1361 {
1362         u16 preset = 0;
1363
1364         switch (host->timing) {
1365         case MMC_TIMING_UHS_SDR12:
1366                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1367                 break;
1368         case MMC_TIMING_UHS_SDR25:
1369                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1370                 break;
1371         case MMC_TIMING_UHS_SDR50:
1372                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1373                 break;
1374         case MMC_TIMING_UHS_SDR104:
1375         case MMC_TIMING_MMC_HS200:
1376                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1377                 break;
1378         case MMC_TIMING_UHS_DDR50:
1379         case MMC_TIMING_MMC_DDR52:
1380                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1381                 break;
1382         case MMC_TIMING_MMC_HS400:
1383                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1384                 break;
1385         default:
1386                 pr_warn("%s: Invalid UHS-I mode selected\n",
1387                         mmc_hostname(host->mmc));
1388                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1389                 break;
1390         }
1391         return preset;
1392 }
1393
1394 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1395                    unsigned int *actual_clock)
1396 {
1397         int div = 0; /* Initialized for compiler warning */
1398         int real_div = div, clk_mul = 1;
1399         u16 clk = 0;
1400         bool switch_base_clk = false;
1401
1402         if (host->version >= SDHCI_SPEC_300) {
1403                 if (host->preset_enabled) {
1404                         u16 pre_val;
1405
1406                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1407                         pre_val = sdhci_get_preset_value(host);
1408                         div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1409                                 >> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1410                         if (host->clk_mul &&
1411                                 (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1412                                 clk = SDHCI_PROG_CLOCK_MODE;
1413                                 real_div = div + 1;
1414                                 clk_mul = host->clk_mul;
1415                         } else {
1416                                 real_div = max_t(int, 1, div << 1);
1417                         }
1418                         goto clock_set;
1419                 }
1420
1421                 /*
1422                  * Check if the Host Controller supports Programmable Clock
1423                  * Mode.
1424                  */
1425                 if (host->clk_mul) {
1426                         for (div = 1; div <= 1024; div++) {
1427                                 if ((host->max_clk * host->clk_mul / div)
1428                                         <= clock)
1429                                         break;
1430                         }
1431                         if ((host->max_clk * host->clk_mul / div) <= clock) {
1432                                 /*
1433                                  * Set Programmable Clock Mode in the Clock
1434                                  * Control register.
1435                                  */
1436                                 clk = SDHCI_PROG_CLOCK_MODE;
1437                                 real_div = div;
1438                                 clk_mul = host->clk_mul;
1439                                 div--;
1440                         } else {
1441                                 /*
1442                                  * Divisor can be too small to reach clock
1443                                  * speed requirement. Then use the base clock.
1444                                  */
1445                                 switch_base_clk = true;
1446                         }
1447                 }
1448
1449                 if (!host->clk_mul || switch_base_clk) {
1450                         /* Version 3.00 divisors must be a multiple of 2. */
1451                         if (host->max_clk <= clock)
1452                                 div = 1;
1453                         else {
1454                                 for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1455                                      div += 2) {
1456                                         if ((host->max_clk / div) <= clock)
1457                                                 break;
1458                                 }
1459                         }
1460                         real_div = div;
1461                         div >>= 1;
1462                         if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1463                                 && !div && host->max_clk <= 25000000)
1464                                 div = 1;
1465                 }
1466         } else {
1467                 /* Version 2.00 divisors must be a power of 2. */
1468                 for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1469                         if ((host->max_clk / div) <= clock)
1470                                 break;
1471                 }
1472                 real_div = div;
1473                 div >>= 1;
1474         }
1475
1476 clock_set:
1477         if (real_div)
1478                 *actual_clock = (host->max_clk * clk_mul) / real_div;
1479         clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1480         clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1481                 << SDHCI_DIVIDER_HI_SHIFT;
1482
1483         return clk;
1484 }
1485 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1486
1487 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1488 {
1489         ktime_t timeout;
1490
1491         clk |= SDHCI_CLOCK_INT_EN;
1492         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1493
1494         /* Wait max 20 ms */
1495         timeout = ktime_add_ms(ktime_get(), 20);
1496         while (!((clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL))
1497                 & SDHCI_CLOCK_INT_STABLE)) {
1498                 if (ktime_after(ktime_get(), timeout)) {
1499                         pr_err("%s: Internal clock never stabilised.\n",
1500                                mmc_hostname(host->mmc));
1501                         sdhci_dumpregs(host);
1502                         return;
1503                 }
1504                 udelay(10);
1505         }
1506
1507         clk |= SDHCI_CLOCK_CARD_EN;
1508         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1509 }
1510 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1511
1512 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1513 {
1514         u16 clk;
1515
1516         host->mmc->actual_clock = 0;
1517
1518         sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1519
1520         if (clock == 0)
1521                 return;
1522
1523         clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1524         sdhci_enable_clk(host, clk);
1525 }
1526 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1527
1528 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1529                                 unsigned short vdd)
1530 {
1531         struct mmc_host *mmc = host->mmc;
1532
1533         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1534
1535         if (mode != MMC_POWER_OFF)
1536                 sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1537         else
1538                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1539 }
1540
1541 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1542                            unsigned short vdd)
1543 {
1544         u8 pwr = 0;
1545
1546         if (mode != MMC_POWER_OFF) {
1547                 switch (1 << vdd) {
1548                 case MMC_VDD_165_195:
1549                 /*
1550                  * Without a regulator, SDHCI does not support 2.0v
1551                  * so we only get here if the driver deliberately
1552                  * added the 2.0v range to ocr_avail. Map it to 1.8v
1553                  * for the purpose of turning on the power.
1554                  */
1555                 case MMC_VDD_20_21:
1556                         pwr = SDHCI_POWER_180;
1557                         break;
1558                 case MMC_VDD_29_30:
1559                 case MMC_VDD_30_31:
1560                         pwr = SDHCI_POWER_300;
1561                         break;
1562                 case MMC_VDD_32_33:
1563                 case MMC_VDD_33_34:
1564                         pwr = SDHCI_POWER_330;
1565                         break;
1566                 default:
1567                         WARN(1, "%s: Invalid vdd %#x\n",
1568                              mmc_hostname(host->mmc), vdd);
1569                         break;
1570                 }
1571         }
1572
1573         if (host->pwr == pwr)
1574                 return;
1575
1576         host->pwr = pwr;
1577
1578         if (pwr == 0) {
1579                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1580                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1581                         sdhci_runtime_pm_bus_off(host);
1582         } else {
1583                 /*
1584                  * Spec says that we should clear the power reg before setting
1585                  * a new value. Some controllers don't seem to like this though.
1586                  */
1587                 if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1588                         sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1589
1590                 /*
1591                  * At least the Marvell CaFe chip gets confused if we set the
1592                  * voltage and set turn on power at the same time, so set the
1593                  * voltage first.
1594                  */
1595                 if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1596                         sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1597
1598                 pwr |= SDHCI_POWER_ON;
1599
1600                 sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1601
1602                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1603                         sdhci_runtime_pm_bus_on(host);
1604
1605                 /*
1606                  * Some controllers need an extra 10ms delay of 10ms before
1607                  * they can apply clock after applying power
1608                  */
1609                 if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1610                         mdelay(10);
1611         }
1612 }
1613 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1614
1615 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1616                      unsigned short vdd)
1617 {
1618         if (IS_ERR(host->mmc->supply.vmmc))
1619                 sdhci_set_power_noreg(host, mode, vdd);
1620         else
1621                 sdhci_set_power_reg(host, mode, vdd);
1622 }
1623 EXPORT_SYMBOL_GPL(sdhci_set_power);
1624
1625 /*****************************************************************************\
1626  *                                                                           *
1627  * MMC callbacks                                                             *
1628  *                                                                           *
1629 \*****************************************************************************/
1630
1631 static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1632 {
1633         struct sdhci_host *host;
1634         int present;
1635         unsigned long flags;
1636
1637         host = mmc_priv(mmc);
1638
1639         /* Firstly check card presence */
1640         present = mmc->ops->get_cd(mmc);
1641
1642         spin_lock_irqsave(&host->lock, flags);
1643
1644         sdhci_led_activate(host);
1645
1646         /*
1647          * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1648          * requests if Auto-CMD12 is enabled.
1649          */
1650         if (sdhci_auto_cmd12(host, mrq)) {
1651                 if (mrq->stop) {
1652                         mrq->data->stop = NULL;
1653                         mrq->stop = NULL;
1654                 }
1655         }
1656
1657         if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1658                 mrq->cmd->error = -ENOMEDIUM;
1659                 sdhci_finish_mrq(host, mrq);
1660         } else {
1661                 if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1662                         sdhci_send_command(host, mrq->sbc);
1663                 else
1664                         sdhci_send_command(host, mrq->cmd);
1665         }
1666
1667         mmiowb();
1668         spin_unlock_irqrestore(&host->lock, flags);
1669 }
1670
1671 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1672 {
1673         u8 ctrl;
1674
1675         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1676         if (width == MMC_BUS_WIDTH_8) {
1677                 ctrl &= ~SDHCI_CTRL_4BITBUS;
1678                 ctrl |= SDHCI_CTRL_8BITBUS;
1679         } else {
1680                 if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1681                         ctrl &= ~SDHCI_CTRL_8BITBUS;
1682                 if (width == MMC_BUS_WIDTH_4)
1683                         ctrl |= SDHCI_CTRL_4BITBUS;
1684                 else
1685                         ctrl &= ~SDHCI_CTRL_4BITBUS;
1686         }
1687         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1688 }
1689 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1690
1691 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1692 {
1693         u16 ctrl_2;
1694
1695         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1696         /* Select Bus Speed Mode for host */
1697         ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1698         if ((timing == MMC_TIMING_MMC_HS200) ||
1699             (timing == MMC_TIMING_UHS_SDR104))
1700                 ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1701         else if (timing == MMC_TIMING_UHS_SDR12)
1702                 ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1703         else if (timing == MMC_TIMING_UHS_SDR25)
1704                 ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1705         else if (timing == MMC_TIMING_UHS_SDR50)
1706                 ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1707         else if ((timing == MMC_TIMING_UHS_DDR50) ||
1708                  (timing == MMC_TIMING_MMC_DDR52))
1709                 ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1710         else if (timing == MMC_TIMING_MMC_HS400)
1711                 ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1712         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1713 }
1714 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1715
1716 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1717 {
1718         struct sdhci_host *host = mmc_priv(mmc);
1719         u8 ctrl;
1720
1721         if (ios->power_mode == MMC_POWER_UNDEFINED)
1722                 return;
1723
1724         if (host->flags & SDHCI_DEVICE_DEAD) {
1725                 if (!IS_ERR(mmc->supply.vmmc) &&
1726                     ios->power_mode == MMC_POWER_OFF)
1727                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1728                 return;
1729         }
1730
1731         /*
1732          * Reset the chip on each power off.
1733          * Should clear out any weird states.
1734          */
1735         if (ios->power_mode == MMC_POWER_OFF) {
1736                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1737                 sdhci_reinit(host);
1738         }
1739
1740         if (host->version >= SDHCI_SPEC_300 &&
1741                 (ios->power_mode == MMC_POWER_UP) &&
1742                 !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1743                 sdhci_enable_preset_value(host, false);
1744
1745         if (!ios->clock || ios->clock != host->clock) {
1746                 host->ops->set_clock(host, ios->clock);
1747                 host->clock = ios->clock;
1748
1749                 if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1750                     host->clock) {
1751                         host->timeout_clk = host->mmc->actual_clock ?
1752                                                 host->mmc->actual_clock / 1000 :
1753                                                 host->clock / 1000;
1754                         host->mmc->max_busy_timeout =
1755                                 host->ops->get_max_timeout_count ?
1756                                 host->ops->get_max_timeout_count(host) :
1757                                 1 << 27;
1758                         host->mmc->max_busy_timeout /= host->timeout_clk;
1759                 }
1760         }
1761
1762         if (host->ops->set_power)
1763                 host->ops->set_power(host, ios->power_mode, ios->vdd);
1764         else
1765                 sdhci_set_power(host, ios->power_mode, ios->vdd);
1766
1767         if (host->ops->platform_send_init_74_clocks)
1768                 host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1769
1770         host->ops->set_bus_width(host, ios->bus_width);
1771
1772         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1773
1774         if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1775                 if (ios->timing == MMC_TIMING_SD_HS ||
1776                      ios->timing == MMC_TIMING_MMC_HS ||
1777                      ios->timing == MMC_TIMING_MMC_HS400 ||
1778                      ios->timing == MMC_TIMING_MMC_HS200 ||
1779                      ios->timing == MMC_TIMING_MMC_DDR52 ||
1780                      ios->timing == MMC_TIMING_UHS_SDR50 ||
1781                      ios->timing == MMC_TIMING_UHS_SDR104 ||
1782                      ios->timing == MMC_TIMING_UHS_DDR50 ||
1783                      ios->timing == MMC_TIMING_UHS_SDR25)
1784                         ctrl |= SDHCI_CTRL_HISPD;
1785                 else
1786                         ctrl &= ~SDHCI_CTRL_HISPD;
1787         }
1788
1789         if (host->version >= SDHCI_SPEC_300) {
1790                 u16 clk, ctrl_2;
1791
1792                 if (!host->preset_enabled) {
1793                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1794                         /*
1795                          * We only need to set Driver Strength if the
1796                          * preset value enable is not set.
1797                          */
1798                         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1799                         ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1800                         if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1801                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1802                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1803                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1804                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1805                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1806                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1807                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1808                         else {
1809                                 pr_warn("%s: invalid driver type, default to driver type B\n",
1810                                         mmc_hostname(mmc));
1811                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1812                         }
1813
1814                         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1815                 } else {
1816                         /*
1817                          * According to SDHC Spec v3.00, if the Preset Value
1818                          * Enable in the Host Control 2 register is set, we
1819                          * need to reset SD Clock Enable before changing High
1820                          * Speed Enable to avoid generating clock gliches.
1821                          */
1822
1823                         /* Reset SD Clock Enable */
1824                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1825                         clk &= ~SDHCI_CLOCK_CARD_EN;
1826                         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1827
1828                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1829
1830                         /* Re-enable SD Clock */
1831                         host->ops->set_clock(host, host->clock);
1832                 }
1833
1834                 /* Reset SD Clock Enable */
1835                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1836                 clk &= ~SDHCI_CLOCK_CARD_EN;
1837                 sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1838
1839                 host->ops->set_uhs_signaling(host, ios->timing);
1840                 host->timing = ios->timing;
1841
1842                 if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1843                                 ((ios->timing == MMC_TIMING_UHS_SDR12) ||
1844                                  (ios->timing == MMC_TIMING_UHS_SDR25) ||
1845                                  (ios->timing == MMC_TIMING_UHS_SDR50) ||
1846                                  (ios->timing == MMC_TIMING_UHS_SDR104) ||
1847                                  (ios->timing == MMC_TIMING_UHS_DDR50) ||
1848                                  (ios->timing == MMC_TIMING_MMC_DDR52))) {
1849                         u16 preset;
1850
1851                         sdhci_enable_preset_value(host, true);
1852                         preset = sdhci_get_preset_value(host);
1853                         ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1854                                 >> SDHCI_PRESET_DRV_SHIFT;
1855                 }
1856
1857                 /* Re-enable SD Clock */
1858                 host->ops->set_clock(host, host->clock);
1859         } else
1860                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1861
1862         /*
1863          * Some (ENE) controllers go apeshit on some ios operation,
1864          * signalling timeout and CRC errors even on CMD0. Resetting
1865          * it on each ios seems to solve the problem.
1866          */
1867         if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
1868                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1869
1870         mmiowb();
1871 }
1872 EXPORT_SYMBOL_GPL(sdhci_set_ios);
1873
1874 static int sdhci_get_cd(struct mmc_host *mmc)
1875 {
1876         struct sdhci_host *host = mmc_priv(mmc);
1877         int gpio_cd = mmc_gpio_get_cd(mmc);
1878
1879         if (host->flags & SDHCI_DEVICE_DEAD)
1880                 return 0;
1881
1882         /* If nonremovable, assume that the card is always present. */
1883         if (!mmc_card_is_removable(host->mmc))
1884                 return 1;
1885
1886         /*
1887          * Try slot gpio detect, if defined it take precedence
1888          * over build in controller functionality
1889          */
1890         if (gpio_cd >= 0)
1891                 return !!gpio_cd;
1892
1893         /* If polling, assume that the card is always present. */
1894         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
1895                 return 1;
1896
1897         /* Host native card detect */
1898         return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1899 }
1900
1901 static int sdhci_check_ro(struct sdhci_host *host)
1902 {
1903         unsigned long flags;
1904         int is_readonly;
1905
1906         spin_lock_irqsave(&host->lock, flags);
1907
1908         if (host->flags & SDHCI_DEVICE_DEAD)
1909                 is_readonly = 0;
1910         else if (host->ops->get_ro)
1911                 is_readonly = host->ops->get_ro(host);
1912         else
1913                 is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
1914                                 & SDHCI_WRITE_PROTECT);
1915
1916         spin_unlock_irqrestore(&host->lock, flags);
1917
1918         /* This quirk needs to be replaced by a callback-function later */
1919         return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
1920                 !is_readonly : is_readonly;
1921 }
1922
1923 #define SAMPLE_COUNT    5
1924
1925 static int sdhci_get_ro(struct mmc_host *mmc)
1926 {
1927         struct sdhci_host *host = mmc_priv(mmc);
1928         int i, ro_count;
1929
1930         if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
1931                 return sdhci_check_ro(host);
1932
1933         ro_count = 0;
1934         for (i = 0; i < SAMPLE_COUNT; i++) {
1935                 if (sdhci_check_ro(host)) {
1936                         if (++ro_count > SAMPLE_COUNT / 2)
1937                                 return 1;
1938                 }
1939                 msleep(30);
1940         }
1941         return 0;
1942 }
1943
1944 static void sdhci_hw_reset(struct mmc_host *mmc)
1945 {
1946         struct sdhci_host *host = mmc_priv(mmc);
1947
1948         if (host->ops && host->ops->hw_reset)
1949                 host->ops->hw_reset(host);
1950 }
1951
1952 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
1953 {
1954         if (!(host->flags & SDHCI_DEVICE_DEAD)) {
1955                 if (enable)
1956                         host->ier |= SDHCI_INT_CARD_INT;
1957                 else
1958                         host->ier &= ~SDHCI_INT_CARD_INT;
1959
1960                 sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1961                 sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1962                 mmiowb();
1963         }
1964 }
1965
1966 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1967 {
1968         struct sdhci_host *host = mmc_priv(mmc);
1969         unsigned long flags;
1970
1971         if (enable)
1972                 pm_runtime_get_noresume(host->mmc->parent);
1973
1974         spin_lock_irqsave(&host->lock, flags);
1975         if (enable)
1976                 host->flags |= SDHCI_SDIO_IRQ_ENABLED;
1977         else
1978                 host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
1979
1980         sdhci_enable_sdio_irq_nolock(host, enable);
1981         spin_unlock_irqrestore(&host->lock, flags);
1982
1983         if (!enable)
1984                 pm_runtime_put_noidle(host->mmc->parent);
1985 }
1986 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
1987
1988 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
1989                                       struct mmc_ios *ios)
1990 {
1991         struct sdhci_host *host = mmc_priv(mmc);
1992         u16 ctrl;
1993         int ret;
1994
1995         /*
1996          * Signal Voltage Switching is only applicable for Host Controllers
1997          * v3.00 and above.
1998          */
1999         if (host->version < SDHCI_SPEC_300)
2000                 return 0;
2001
2002         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2003
2004         switch (ios->signal_voltage) {
2005         case MMC_SIGNAL_VOLTAGE_330:
2006                 if (!(host->flags & SDHCI_SIGNALING_330))
2007                         return -EINVAL;
2008                 /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
2009                 ctrl &= ~SDHCI_CTRL_VDD_180;
2010                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2011
2012                 if (!IS_ERR(mmc->supply.vqmmc)) {
2013                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2014                         if (ret) {
2015                                 pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
2016                                         mmc_hostname(mmc));
2017                                 return -EIO;
2018                         }
2019                 }
2020                 /* Wait for 5ms */
2021                 usleep_range(5000, 5500);
2022
2023                 /* 3.3V regulator output should be stable within 5 ms */
2024                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2025                 if (!(ctrl & SDHCI_CTRL_VDD_180))
2026                         return 0;
2027
2028                 pr_warn("%s: 3.3V regulator output did not became stable\n",
2029                         mmc_hostname(mmc));
2030
2031                 return -EAGAIN;
2032         case MMC_SIGNAL_VOLTAGE_180:
2033                 if (!(host->flags & SDHCI_SIGNALING_180))
2034                         return -EINVAL;
2035                 if (!IS_ERR(mmc->supply.vqmmc)) {
2036                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2037                         if (ret) {
2038                                 pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
2039                                         mmc_hostname(mmc));
2040                                 return -EIO;
2041                         }
2042                 }
2043
2044                 /*
2045                  * Enable 1.8V Signal Enable in the Host Control2
2046                  * register
2047                  */
2048                 ctrl |= SDHCI_CTRL_VDD_180;
2049                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2050
2051                 /* Some controller need to do more when switching */
2052                 if (host->ops->voltage_switch)
2053                         host->ops->voltage_switch(host);
2054
2055                 /* 1.8V regulator output should be stable within 5 ms */
2056                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2057                 if (ctrl & SDHCI_CTRL_VDD_180)
2058                         return 0;
2059
2060                 pr_warn("%s: 1.8V regulator output did not became stable\n",
2061                         mmc_hostname(mmc));
2062
2063                 return -EAGAIN;
2064         case MMC_SIGNAL_VOLTAGE_120:
2065                 if (!(host->flags & SDHCI_SIGNALING_120))
2066                         return -EINVAL;
2067                 if (!IS_ERR(mmc->supply.vqmmc)) {
2068                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2069                         if (ret) {
2070                                 pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2071                                         mmc_hostname(mmc));
2072                                 return -EIO;
2073                         }
2074                 }
2075                 return 0;
2076         default:
2077                 /* No signal voltage switch required */
2078                 return 0;
2079         }
2080 }
2081 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2082
2083 static int sdhci_card_busy(struct mmc_host *mmc)
2084 {
2085         struct sdhci_host *host = mmc_priv(mmc);
2086         u32 present_state;
2087
2088         /* Check whether DAT[0] is 0 */
2089         present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2090
2091         return !(present_state & SDHCI_DATA_0_LVL_MASK);
2092 }
2093
2094 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2095 {
2096         struct sdhci_host *host = mmc_priv(mmc);
2097         unsigned long flags;
2098
2099         spin_lock_irqsave(&host->lock, flags);
2100         host->flags |= SDHCI_HS400_TUNING;
2101         spin_unlock_irqrestore(&host->lock, flags);
2102
2103         return 0;
2104 }
2105
2106 static void sdhci_start_tuning(struct sdhci_host *host)
2107 {
2108         u16 ctrl;
2109
2110         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2111         ctrl |= SDHCI_CTRL_EXEC_TUNING;
2112         if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2113                 ctrl |= SDHCI_CTRL_TUNED_CLK;
2114         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2115
2116         /*
2117          * As per the Host Controller spec v3.00, tuning command
2118          * generates Buffer Read Ready interrupt, so enable that.
2119          *
2120          * Note: The spec clearly says that when tuning sequence
2121          * is being performed, the controller does not generate
2122          * interrupts other than Buffer Read Ready interrupt. But
2123          * to make sure we don't hit a controller bug, we _only_
2124          * enable Buffer Read Ready interrupt here.
2125          */
2126         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2127         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2128 }
2129
2130 static void sdhci_end_tuning(struct sdhci_host *host)
2131 {
2132         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2133         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2134 }
2135
2136 static void sdhci_reset_tuning(struct sdhci_host *host)
2137 {
2138         u16 ctrl;
2139
2140         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2141         ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2142         ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2143         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2144 }
2145
2146 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2147 {
2148         sdhci_reset_tuning(host);
2149
2150         sdhci_do_reset(host, SDHCI_RESET_CMD);
2151         sdhci_do_reset(host, SDHCI_RESET_DATA);
2152
2153         sdhci_end_tuning(host);
2154
2155         mmc_abort_tuning(host->mmc, opcode);
2156 }
2157
2158 /*
2159  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2160  * tuning command does not have a data payload (or rather the hardware does it
2161  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2162  * interrupt setup is different to other commands and there is no timeout
2163  * interrupt so special handling is needed.
2164  */
2165 static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2166 {
2167         struct mmc_host *mmc = host->mmc;
2168         struct mmc_command cmd = {};
2169         struct mmc_request mrq = {};
2170         unsigned long flags;
2171         u32 b = host->sdma_boundary;
2172
2173         spin_lock_irqsave(&host->lock, flags);
2174
2175         cmd.opcode = opcode;
2176         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2177         cmd.mrq = &mrq;
2178
2179         mrq.cmd = &cmd;
2180         /*
2181          * In response to CMD19, the card sends 64 bytes of tuning
2182          * block to the Host Controller. So we set the block size
2183          * to 64 here.
2184          */
2185         if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2186             mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2187                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2188         else
2189                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2190
2191         /*
2192          * The tuning block is sent by the card to the host controller.
2193          * So we set the TRNS_READ bit in the Transfer Mode register.
2194          * This also takes care of setting DMA Enable and Multi Block
2195          * Select in the same register to 0.
2196          */
2197         sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2198
2199         sdhci_send_command(host, &cmd);
2200
2201         host->cmd = NULL;
2202
2203         sdhci_del_timer(host, &mrq);
2204
2205         host->tuning_done = 0;
2206
2207         mmiowb();
2208         spin_unlock_irqrestore(&host->lock, flags);
2209
2210         /* Wait for Buffer Read Ready interrupt */
2211         wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2212                            msecs_to_jiffies(50));
2213
2214 }
2215
2216 static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2217 {
2218         int i;
2219
2220         /*
2221          * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2222          * of loops reaches 40 times.
2223          */
2224         for (i = 0; i < MAX_TUNING_LOOP; i++) {
2225                 u16 ctrl;
2226
2227                 sdhci_send_tuning(host, opcode);
2228
2229                 if (!host->tuning_done) {
2230                         pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
2231                                 mmc_hostname(host->mmc));
2232                         sdhci_abort_tuning(host, opcode);
2233                         return;
2234                 }
2235
2236                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2237                 if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2238                         if (ctrl & SDHCI_CTRL_TUNED_CLK)
2239                                 return; /* Success! */
2240                         break;
2241                 }
2242
2243                 /* Spec does not require a delay between tuning cycles */
2244                 if (host->tuning_delay > 0)
2245                         mdelay(host->tuning_delay);
2246         }
2247
2248         pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2249                 mmc_hostname(host->mmc));
2250         sdhci_reset_tuning(host);
2251 }
2252
2253 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2254 {
2255         struct sdhci_host *host = mmc_priv(mmc);
2256         int err = 0;
2257         unsigned int tuning_count = 0;
2258         bool hs400_tuning;
2259
2260         hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2261
2262         if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2263                 tuning_count = host->tuning_count;
2264
2265         /*
2266          * The Host Controller needs tuning in case of SDR104 and DDR50
2267          * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2268          * the Capabilities register.
2269          * If the Host Controller supports the HS200 mode then the
2270          * tuning function has to be executed.
2271          */
2272         switch (host->timing) {
2273         /* HS400 tuning is done in HS200 mode */
2274         case MMC_TIMING_MMC_HS400:
2275                 err = -EINVAL;
2276                 goto out;
2277
2278         case MMC_TIMING_MMC_HS200:
2279                 /*
2280                  * Periodic re-tuning for HS400 is not expected to be needed, so
2281                  * disable it here.
2282                  */
2283                 if (hs400_tuning)
2284                         tuning_count = 0;
2285                 break;
2286
2287         case MMC_TIMING_UHS_SDR104:
2288         case MMC_TIMING_UHS_DDR50:
2289                 break;
2290
2291         case MMC_TIMING_UHS_SDR50:
2292                 if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2293                         break;
2294                 /* FALLTHROUGH */
2295
2296         default:
2297                 goto out;
2298         }
2299
2300         if (host->ops->platform_execute_tuning) {
2301                 err = host->ops->platform_execute_tuning(host, opcode);
2302                 goto out;
2303         }
2304
2305         host->mmc->retune_period = tuning_count;
2306
2307         if (host->tuning_delay < 0)
2308                 host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2309
2310         sdhci_start_tuning(host);
2311
2312         __sdhci_execute_tuning(host, opcode);
2313
2314         sdhci_end_tuning(host);
2315 out:
2316         host->flags &= ~SDHCI_HS400_TUNING;
2317
2318         return err;
2319 }
2320 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2321
2322 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2323 {
2324         /* Host Controller v3.00 defines preset value registers */
2325         if (host->version < SDHCI_SPEC_300)
2326                 return;
2327
2328         /*
2329          * We only enable or disable Preset Value if they are not already
2330          * enabled or disabled respectively. Otherwise, we bail out.
2331          */
2332         if (host->preset_enabled != enable) {
2333                 u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2334
2335                 if (enable)
2336                         ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2337                 else
2338                         ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2339
2340                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2341
2342                 if (enable)
2343                         host->flags |= SDHCI_PV_ENABLED;
2344                 else
2345                         host->flags &= ~SDHCI_PV_ENABLED;
2346
2347                 host->preset_enabled = enable;
2348         }
2349 }
2350
2351 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2352                                 int err)
2353 {
2354         struct sdhci_host *host = mmc_priv(mmc);
2355         struct mmc_data *data = mrq->data;
2356
2357         if (data->host_cookie != COOKIE_UNMAPPED)
2358                 dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2359                              mmc_get_dma_dir(data));
2360
2361         data->host_cookie = COOKIE_UNMAPPED;
2362 }
2363
2364 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2365 {
2366         struct sdhci_host *host = mmc_priv(mmc);
2367
2368         mrq->data->host_cookie = COOKIE_UNMAPPED;
2369
2370         /*
2371          * No pre-mapping in the pre hook if we're using the bounce buffer,
2372          * for that we would need two bounce buffers since one buffer is
2373          * in flight when this is getting called.
2374          */
2375         if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2376                 sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2377 }
2378
2379 static inline bool sdhci_has_requests(struct sdhci_host *host)
2380 {
2381         return host->cmd || host->data_cmd;
2382 }
2383
2384 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2385 {
2386         if (host->data_cmd) {
2387                 host->data_cmd->error = err;
2388                 sdhci_finish_mrq(host, host->data_cmd->mrq);
2389         }
2390
2391         if (host->cmd) {
2392                 host->cmd->error = err;
2393                 sdhci_finish_mrq(host, host->cmd->mrq);
2394         }
2395 }
2396
2397 static void sdhci_card_event(struct mmc_host *mmc)
2398 {
2399         struct sdhci_host *host = mmc_priv(mmc);
2400         unsigned long flags;
2401         int present;
2402
2403         /* First check if client has provided their own card event */
2404         if (host->ops->card_event)
2405                 host->ops->card_event(host);
2406
2407         present = mmc->ops->get_cd(mmc);
2408
2409         spin_lock_irqsave(&host->lock, flags);
2410
2411         /* Check sdhci_has_requests() first in case we are runtime suspended */
2412         if (sdhci_has_requests(host) && !present) {
2413                 pr_err("%s: Card removed during transfer!\n",
2414                         mmc_hostname(host->mmc));
2415                 pr_err("%s: Resetting controller.\n",
2416                         mmc_hostname(host->mmc));
2417
2418                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2419                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2420
2421                 sdhci_error_out_mrqs(host, -ENOMEDIUM);
2422         }
2423
2424         spin_unlock_irqrestore(&host->lock, flags);
2425 }
2426
2427 static const struct mmc_host_ops sdhci_ops = {
2428         .request        = sdhci_request,
2429         .post_req       = sdhci_post_req,
2430         .pre_req        = sdhci_pre_req,
2431         .set_ios        = sdhci_set_ios,
2432         .get_cd         = sdhci_get_cd,
2433         .get_ro         = sdhci_get_ro,
2434         .hw_reset       = sdhci_hw_reset,
2435         .enable_sdio_irq = sdhci_enable_sdio_irq,
2436         .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2437         .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2438         .execute_tuning                 = sdhci_execute_tuning,
2439         .card_event                     = sdhci_card_event,
2440         .card_busy      = sdhci_card_busy,
2441 };
2442
2443 /*****************************************************************************\
2444  *                                                                           *
2445  * Tasklets                                                                  *
2446  *                                                                           *
2447 \*****************************************************************************/
2448
2449 static bool sdhci_request_done(struct sdhci_host *host)
2450 {
2451         unsigned long flags;
2452         struct mmc_request *mrq;
2453         int i;
2454
2455         spin_lock_irqsave(&host->lock, flags);
2456
2457         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2458                 mrq = host->mrqs_done[i];
2459                 if (mrq)
2460                         break;
2461         }
2462
2463         if (!mrq) {
2464                 spin_unlock_irqrestore(&host->lock, flags);
2465                 return true;
2466         }
2467
2468         sdhci_del_timer(host, mrq);
2469
2470         /*
2471          * Always unmap the data buffers if they were mapped by
2472          * sdhci_prepare_data() whenever we finish with a request.
2473          * This avoids leaking DMA mappings on error.
2474          */
2475         if (host->flags & SDHCI_REQ_USE_DMA) {
2476                 struct mmc_data *data = mrq->data;
2477
2478                 if (data && data->host_cookie == COOKIE_MAPPED) {
2479                         if (host->bounce_buffer) {
2480                                 /*
2481                                  * On reads, copy the bounced data into the
2482                                  * sglist
2483                                  */
2484                                 if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2485                                         unsigned int length = data->bytes_xfered;
2486
2487                                         if (length > host->bounce_buffer_size) {
2488                                                 pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2489                                                        mmc_hostname(host->mmc),
2490                                                        host->bounce_buffer_size,
2491                                                        data->bytes_xfered);
2492                                                 /* Cap it down and continue */
2493                                                 length = host->bounce_buffer_size;
2494                                         }
2495                                         dma_sync_single_for_cpu(
2496                                                 host->mmc->parent,
2497                                                 host->bounce_addr,
2498                                                 host->bounce_buffer_size,
2499                                                 DMA_FROM_DEVICE);
2500                                         sg_copy_from_buffer(data->sg,
2501                                                 data->sg_len,
2502                                                 host->bounce_buffer,
2503                                                 length);
2504                                 } else {
2505                                         /* No copying, just switch ownership */
2506                                         dma_sync_single_for_cpu(
2507                                                 host->mmc->parent,
2508                                                 host->bounce_addr,
2509                                                 host->bounce_buffer_size,
2510                                                 mmc_get_dma_dir(data));
2511                                 }
2512                         } else {
2513                                 /* Unmap the raw data */
2514                                 dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2515                                              data->sg_len,
2516                                              mmc_get_dma_dir(data));
2517                         }
2518                         data->host_cookie = COOKIE_UNMAPPED;
2519                 }
2520         }
2521
2522         /*
2523          * The controller needs a reset of internal state machines
2524          * upon error conditions.
2525          */
2526         if (sdhci_needs_reset(host, mrq)) {
2527                 /*
2528                  * Do not finish until command and data lines are available for
2529                  * reset. Note there can only be one other mrq, so it cannot
2530                  * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2531                  * would both be null.
2532                  */
2533                 if (host->cmd || host->data_cmd) {
2534                         spin_unlock_irqrestore(&host->lock, flags);
2535                         return true;
2536                 }
2537
2538                 /* Some controllers need this kick or reset won't work here */
2539                 if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2540                         /* This is to force an update */
2541                         host->ops->set_clock(host, host->clock);
2542
2543                 /* Spec says we should do both at the same time, but Ricoh
2544                    controllers do not like that. */
2545                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2546                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2547
2548                 host->pending_reset = false;
2549         }
2550
2551         if (!sdhci_has_requests(host))
2552                 sdhci_led_deactivate(host);
2553
2554         host->mrqs_done[i] = NULL;
2555
2556         mmiowb();
2557         spin_unlock_irqrestore(&host->lock, flags);
2558
2559         mmc_request_done(host->mmc, mrq);
2560
2561         return false;
2562 }
2563
2564 static void sdhci_tasklet_finish(unsigned long param)
2565 {
2566         struct sdhci_host *host = (struct sdhci_host *)param;
2567
2568         while (!sdhci_request_done(host))
2569                 ;
2570 }
2571
2572 static void sdhci_timeout_timer(struct timer_list *t)
2573 {
2574         struct sdhci_host *host;
2575         unsigned long flags;
2576
2577         host = from_timer(host, t, timer);
2578
2579         spin_lock_irqsave(&host->lock, flags);
2580
2581         if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2582                 pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2583                        mmc_hostname(host->mmc));
2584                 sdhci_dumpregs(host);
2585
2586                 host->cmd->error = -ETIMEDOUT;
2587                 sdhci_finish_mrq(host, host->cmd->mrq);
2588         }
2589
2590         mmiowb();
2591         spin_unlock_irqrestore(&host->lock, flags);
2592 }
2593
2594 static void sdhci_timeout_data_timer(struct timer_list *t)
2595 {
2596         struct sdhci_host *host;
2597         unsigned long flags;
2598
2599         host = from_timer(host, t, data_timer);
2600
2601         spin_lock_irqsave(&host->lock, flags);
2602
2603         if (host->data || host->data_cmd ||
2604             (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2605                 pr_err("%s: Timeout waiting for hardware interrupt.\n",
2606                        mmc_hostname(host->mmc));
2607                 sdhci_dumpregs(host);
2608
2609                 if (host->data) {
2610                         host->data->error = -ETIMEDOUT;
2611                         sdhci_finish_data(host);
2612                 } else if (host->data_cmd) {
2613                         host->data_cmd->error = -ETIMEDOUT;
2614                         sdhci_finish_mrq(host, host->data_cmd->mrq);
2615                 } else {
2616                         host->cmd->error = -ETIMEDOUT;
2617                         sdhci_finish_mrq(host, host->cmd->mrq);
2618                 }
2619         }
2620
2621         mmiowb();
2622         spin_unlock_irqrestore(&host->lock, flags);
2623 }
2624
2625 /*****************************************************************************\
2626  *                                                                           *
2627  * Interrupt handling                                                        *
2628  *                                                                           *
2629 \*****************************************************************************/
2630
2631 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask)
2632 {
2633         if (!host->cmd) {
2634                 /*
2635                  * SDHCI recovers from errors by resetting the cmd and data
2636                  * circuits.  Until that is done, there very well might be more
2637                  * interrupts, so ignore them in that case.
2638                  */
2639                 if (host->pending_reset)
2640                         return;
2641                 pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2642                        mmc_hostname(host->mmc), (unsigned)intmask);
2643                 sdhci_dumpregs(host);
2644                 return;
2645         }
2646
2647         if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2648                        SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2649                 if (intmask & SDHCI_INT_TIMEOUT)
2650                         host->cmd->error = -ETIMEDOUT;
2651                 else
2652                         host->cmd->error = -EILSEQ;
2653
2654                 /*
2655                  * If this command initiates a data phase and a response
2656                  * CRC error is signalled, the card can start transferring
2657                  * data - the card may have received the command without
2658                  * error.  We must not terminate the mmc_request early.
2659                  *
2660                  * If the card did not receive the command or returned an
2661                  * error which prevented it sending data, the data phase
2662                  * will time out.
2663                  */
2664                 if (host->cmd->data &&
2665                     (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2666                      SDHCI_INT_CRC) {
2667                         host->cmd = NULL;
2668                         return;
2669                 }
2670
2671                 sdhci_finish_mrq(host, host->cmd->mrq);
2672                 return;
2673         }
2674
2675         if (intmask & SDHCI_INT_RESPONSE)
2676                 sdhci_finish_command(host);
2677 }
2678
2679 static void sdhci_adma_show_error(struct sdhci_host *host)
2680 {
2681         void *desc = host->adma_table;
2682
2683         sdhci_dumpregs(host);
2684
2685         while (true) {
2686                 struct sdhci_adma2_64_desc *dma_desc = desc;
2687
2688                 if (host->flags & SDHCI_USE_64_BIT_DMA)
2689                         DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2690                             desc, le32_to_cpu(dma_desc->addr_hi),
2691                             le32_to_cpu(dma_desc->addr_lo),
2692                             le16_to_cpu(dma_desc->len),
2693                             le16_to_cpu(dma_desc->cmd));
2694                 else
2695                         DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2696                             desc, le32_to_cpu(dma_desc->addr_lo),
2697                             le16_to_cpu(dma_desc->len),
2698                             le16_to_cpu(dma_desc->cmd));
2699
2700                 desc += host->desc_sz;
2701
2702                 if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2703                         break;
2704         }
2705 }
2706
2707 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2708 {
2709         u32 command;
2710
2711         /* CMD19 generates _only_ Buffer Read Ready interrupt */
2712         if (intmask & SDHCI_INT_DATA_AVAIL) {
2713                 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2714                 if (command == MMC_SEND_TUNING_BLOCK ||
2715                     command == MMC_SEND_TUNING_BLOCK_HS200) {
2716                         host->tuning_done = 1;
2717                         wake_up(&host->buf_ready_int);
2718                         return;
2719                 }
2720         }
2721
2722         if (!host->data) {
2723                 struct mmc_command *data_cmd = host->data_cmd;
2724
2725                 /*
2726                  * The "data complete" interrupt is also used to
2727                  * indicate that a busy state has ended. See comment
2728                  * above in sdhci_cmd_irq().
2729                  */
2730                 if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2731                         if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2732                                 host->data_cmd = NULL;
2733                                 data_cmd->error = -ETIMEDOUT;
2734                                 sdhci_finish_mrq(host, data_cmd->mrq);
2735                                 return;
2736                         }
2737                         if (intmask & SDHCI_INT_DATA_END) {
2738                                 host->data_cmd = NULL;
2739                                 /*
2740                                  * Some cards handle busy-end interrupt
2741                                  * before the command completed, so make
2742                                  * sure we do things in the proper order.
2743                                  */
2744                                 if (host->cmd == data_cmd)
2745                                         return;
2746
2747                                 sdhci_finish_mrq(host, data_cmd->mrq);
2748                                 return;
2749                         }
2750                 }
2751
2752                 /*
2753                  * SDHCI recovers from errors by resetting the cmd and data
2754                  * circuits. Until that is done, there very well might be more
2755                  * interrupts, so ignore them in that case.
2756                  */
2757                 if (host->pending_reset)
2758                         return;
2759
2760                 pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2761                        mmc_hostname(host->mmc), (unsigned)intmask);
2762                 sdhci_dumpregs(host);
2763
2764                 return;
2765         }
2766
2767         if (intmask & SDHCI_INT_DATA_TIMEOUT)
2768                 host->data->error = -ETIMEDOUT;
2769         else if (intmask & SDHCI_INT_DATA_END_BIT)
2770                 host->data->error = -EILSEQ;
2771         else if ((intmask & SDHCI_INT_DATA_CRC) &&
2772                 SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2773                         != MMC_BUS_TEST_R)
2774                 host->data->error = -EILSEQ;
2775         else if (intmask & SDHCI_INT_ADMA_ERROR) {
2776                 pr_err("%s: ADMA error\n", mmc_hostname(host->mmc));
2777                 sdhci_adma_show_error(host);
2778                 host->data->error = -EIO;
2779                 if (host->ops->adma_workaround)
2780                         host->ops->adma_workaround(host, intmask);
2781         }
2782
2783         if (host->data->error)
2784                 sdhci_finish_data(host);
2785         else {
2786                 if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2787                         sdhci_transfer_pio(host);
2788
2789                 /*
2790                  * We currently don't do anything fancy with DMA
2791                  * boundaries, but as we can't disable the feature
2792                  * we need to at least restart the transfer.
2793                  *
2794                  * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2795                  * should return a valid address to continue from, but as
2796                  * some controllers are faulty, don't trust them.
2797                  */
2798                 if (intmask & SDHCI_INT_DMA_END) {
2799                         u32 dmastart, dmanow;
2800
2801                         dmastart = sdhci_sdma_address(host);
2802                         dmanow = dmastart + host->data->bytes_xfered;
2803                         /*
2804                          * Force update to the next DMA block boundary.
2805                          */
2806                         dmanow = (dmanow &
2807                                 ~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2808                                 SDHCI_DEFAULT_BOUNDARY_SIZE;
2809                         host->data->bytes_xfered = dmanow - dmastart;
2810                         DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n",
2811                             dmastart, host->data->bytes_xfered, dmanow);
2812                         sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS);
2813                 }
2814
2815                 if (intmask & SDHCI_INT_DATA_END) {
2816                         if (host->cmd == host->data_cmd) {
2817                                 /*
2818                                  * Data managed to finish before the
2819                                  * command completed. Make sure we do
2820                                  * things in the proper order.
2821                                  */
2822                                 host->data_early = 1;
2823                         } else {
2824                                 sdhci_finish_data(host);
2825                         }
2826                 }
2827         }
2828 }
2829
2830 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2831 {
2832         irqreturn_t result = IRQ_NONE;
2833         struct sdhci_host *host = dev_id;
2834         u32 intmask, mask, unexpected = 0;
2835         int max_loops = 16;
2836
2837         spin_lock(&host->lock);
2838
2839         if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
2840                 spin_unlock(&host->lock);
2841                 return IRQ_NONE;
2842         }
2843
2844         intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2845         if (!intmask || intmask == 0xffffffff) {
2846                 result = IRQ_NONE;
2847                 goto out;
2848         }
2849
2850         do {
2851                 DBG("IRQ status 0x%08x\n", intmask);
2852
2853                 if (host->ops->irq) {
2854                         intmask = host->ops->irq(host, intmask);
2855                         if (!intmask)
2856                                 goto cont;
2857                 }
2858
2859                 /* Clear selected interrupts. */
2860                 mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2861                                   SDHCI_INT_BUS_POWER);
2862                 sdhci_writel(host, mask, SDHCI_INT_STATUS);
2863
2864                 if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2865                         u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
2866                                       SDHCI_CARD_PRESENT;
2867
2868                         /*
2869                          * There is a observation on i.mx esdhc.  INSERT
2870                          * bit will be immediately set again when it gets
2871                          * cleared, if a card is inserted.  We have to mask
2872                          * the irq to prevent interrupt storm which will
2873                          * freeze the system.  And the REMOVE gets the
2874                          * same situation.
2875                          *
2876                          * More testing are needed here to ensure it works
2877                          * for other platforms though.
2878                          */
2879                         host->ier &= ~(SDHCI_INT_CARD_INSERT |
2880                                        SDHCI_INT_CARD_REMOVE);
2881                         host->ier |= present ? SDHCI_INT_CARD_REMOVE :
2882                                                SDHCI_INT_CARD_INSERT;
2883                         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2884                         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2885
2886                         sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
2887                                      SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
2888
2889                         host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
2890                                                        SDHCI_INT_CARD_REMOVE);
2891                         result = IRQ_WAKE_THREAD;
2892                 }
2893
2894                 if (intmask & SDHCI_INT_CMD_MASK)
2895                         sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK);
2896
2897                 if (intmask & SDHCI_INT_DATA_MASK)
2898                         sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
2899
2900                 if (intmask & SDHCI_INT_BUS_POWER)
2901                         pr_err("%s: Card is consuming too much power!\n",
2902                                 mmc_hostname(host->mmc));
2903
2904                 if (intmask & SDHCI_INT_RETUNE)
2905                         mmc_retune_needed(host->mmc);
2906
2907                 if ((intmask & SDHCI_INT_CARD_INT) &&
2908                     (host->ier & SDHCI_INT_CARD_INT)) {
2909                         sdhci_enable_sdio_irq_nolock(host, false);
2910                         host->thread_isr |= SDHCI_INT_CARD_INT;
2911                         result = IRQ_WAKE_THREAD;
2912                 }
2913
2914                 intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2915                              SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2916                              SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
2917                              SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
2918
2919                 if (intmask) {
2920                         unexpected |= intmask;
2921                         sdhci_writel(host, intmask, SDHCI_INT_STATUS);
2922                 }
2923 cont:
2924                 if (result == IRQ_NONE)
2925                         result = IRQ_HANDLED;
2926
2927                 intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2928         } while (intmask && --max_loops);
2929 out:
2930         spin_unlock(&host->lock);
2931
2932         if (unexpected) {
2933                 pr_err("%s: Unexpected interrupt 0x%08x.\n",
2934                            mmc_hostname(host->mmc), unexpected);
2935                 sdhci_dumpregs(host);
2936         }
2937
2938         return result;
2939 }
2940
2941 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
2942 {
2943         struct sdhci_host *host = dev_id;
2944         unsigned long flags;
2945         u32 isr;
2946
2947         spin_lock_irqsave(&host->lock, flags);
2948         isr = host->thread_isr;
2949         host->thread_isr = 0;
2950         spin_unlock_irqrestore(&host->lock, flags);
2951
2952         if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2953                 struct mmc_host *mmc = host->mmc;
2954
2955                 mmc->ops->card_event(mmc);
2956                 mmc_detect_change(mmc, msecs_to_jiffies(200));
2957         }
2958
2959         if (isr & SDHCI_INT_CARD_INT) {
2960                 sdio_run_irqs(host->mmc);
2961
2962                 spin_lock_irqsave(&host->lock, flags);
2963                 if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
2964                         sdhci_enable_sdio_irq_nolock(host, true);
2965                 spin_unlock_irqrestore(&host->lock, flags);
2966         }
2967
2968         return isr ? IRQ_HANDLED : IRQ_NONE;
2969 }
2970
2971 /*****************************************************************************\
2972  *                                                                           *
2973  * Suspend/resume                                                            *
2974  *                                                                           *
2975 \*****************************************************************************/
2976
2977 #ifdef CONFIG_PM
2978
2979 static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
2980 {
2981         return mmc_card_is_removable(host->mmc) &&
2982                !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
2983                !mmc_can_gpio_cd(host->mmc);
2984 }
2985
2986 /*
2987  * To enable wakeup events, the corresponding events have to be enabled in
2988  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
2989  * Table' in the SD Host Controller Standard Specification.
2990  * It is useless to restore SDHCI_INT_ENABLE state in
2991  * sdhci_disable_irq_wakeups() since it will be set by
2992  * sdhci_enable_card_detection() or sdhci_init().
2993  */
2994 static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
2995 {
2996         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
2997                   SDHCI_WAKE_ON_INT;
2998         u32 irq_val = 0;
2999         u8 wake_val = 0;
3000         u8 val;
3001
3002         if (sdhci_cd_irq_can_wakeup(host)) {
3003                 wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
3004                 irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
3005         }
3006
3007         if (mmc_card_wake_sdio_irq(host->mmc)) {
3008                 wake_val |= SDHCI_WAKE_ON_INT;
3009                 irq_val |= SDHCI_INT_CARD_INT;
3010         }
3011
3012         if (!irq_val)
3013                 return false;
3014
3015         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3016         val &= ~mask;
3017         val |= wake_val;
3018         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3019
3020         sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
3021
3022         host->irq_wake_enabled = !enable_irq_wake(host->irq);
3023
3024         return host->irq_wake_enabled;
3025 }
3026
3027 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
3028 {
3029         u8 val;
3030         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
3031                         | SDHCI_WAKE_ON_INT;
3032
3033         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3034         val &= ~mask;
3035         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3036
3037         disable_irq_wake(host->irq);
3038
3039         host->irq_wake_enabled = false;
3040 }
3041
3042 int sdhci_suspend_host(struct sdhci_host *host)
3043 {
3044         sdhci_disable_card_detection(host);
3045
3046         mmc_retune_timer_stop(host->mmc);
3047
3048         if (!device_may_wakeup(mmc_dev(host->mmc)) ||
3049             !sdhci_enable_irq_wakeups(host)) {
3050                 host->ier = 0;
3051                 sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3052                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3053                 free_irq(host->irq, host);
3054         }
3055
3056         return 0;
3057 }
3058
3059 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3060
3061 int sdhci_resume_host(struct sdhci_host *host)
3062 {
3063         struct mmc_host *mmc = host->mmc;
3064         int ret = 0;
3065
3066         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3067                 if (host->ops->enable_dma)
3068                         host->ops->enable_dma(host);
3069         }
3070
3071         if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3072             (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3073                 /* Card keeps power but host controller does not */
3074                 sdhci_init(host, 0);
3075                 host->pwr = 0;
3076                 host->clock = 0;
3077                 mmc->ops->set_ios(mmc, &mmc->ios);
3078         } else {
3079                 sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3080                 mmiowb();
3081         }
3082
3083         if (host->irq_wake_enabled) {
3084                 sdhci_disable_irq_wakeups(host);
3085         } else {
3086                 ret = request_threaded_irq(host->irq, sdhci_irq,
3087                                            sdhci_thread_irq, IRQF_SHARED,
3088                                            mmc_hostname(host->mmc), host);
3089                 if (ret)
3090                         return ret;
3091         }
3092
3093         sdhci_enable_card_detection(host);
3094
3095         return ret;
3096 }
3097
3098 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3099
3100 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3101 {
3102         unsigned long flags;
3103
3104         mmc_retune_timer_stop(host->mmc);
3105
3106         spin_lock_irqsave(&host->lock, flags);
3107         host->ier &= SDHCI_INT_CARD_INT;
3108         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3109         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3110         spin_unlock_irqrestore(&host->lock, flags);
3111
3112         synchronize_hardirq(host->irq);
3113
3114         spin_lock_irqsave(&host->lock, flags);
3115         host->runtime_suspended = true;
3116         spin_unlock_irqrestore(&host->lock, flags);
3117
3118         return 0;
3119 }
3120 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3121
3122 int sdhci_runtime_resume_host(struct sdhci_host *host)
3123 {
3124         struct mmc_host *mmc = host->mmc;
3125         unsigned long flags;
3126         int host_flags = host->flags;
3127
3128         if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3129                 if (host->ops->enable_dma)
3130                         host->ops->enable_dma(host);
3131         }
3132
3133         sdhci_init(host, 0);
3134
3135         if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3136             mmc->ios.power_mode != MMC_POWER_OFF) {
3137                 /* Force clock and power re-program */
3138                 host->pwr = 0;
3139                 host->clock = 0;
3140                 mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3141                 mmc->ops->set_ios(mmc, &mmc->ios);
3142
3143                 if ((host_flags & SDHCI_PV_ENABLED) &&
3144                     !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3145                         spin_lock_irqsave(&host->lock, flags);
3146                         sdhci_enable_preset_value(host, true);
3147                         spin_unlock_irqrestore(&host->lock, flags);
3148                 }
3149
3150                 if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3151                     mmc->ops->hs400_enhanced_strobe)
3152                         mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3153         }
3154
3155         spin_lock_irqsave(&host->lock, flags);
3156
3157         host->runtime_suspended = false;
3158
3159         /* Enable SDIO IRQ */
3160         if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3161                 sdhci_enable_sdio_irq_nolock(host, true);
3162
3163         /* Enable Card Detection */
3164         sdhci_enable_card_detection(host);
3165
3166         spin_unlock_irqrestore(&host->lock, flags);
3167
3168         return 0;
3169 }
3170 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3171
3172 #endif /* CONFIG_PM */
3173
3174 /*****************************************************************************\
3175  *                                                                           *
3176  * Command Queue Engine (CQE) helpers                                        *
3177  *                                                                           *
3178 \*****************************************************************************/
3179
3180 void sdhci_cqe_enable(struct mmc_host *mmc)
3181 {
3182         struct sdhci_host *host = mmc_priv(mmc);
3183         unsigned long flags;
3184         u8 ctrl;
3185
3186         spin_lock_irqsave(&host->lock, flags);
3187
3188         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3189         ctrl &= ~SDHCI_CTRL_DMA_MASK;
3190         if (host->flags & SDHCI_USE_64_BIT_DMA)
3191                 ctrl |= SDHCI_CTRL_ADMA64;
3192         else
3193                 ctrl |= SDHCI_CTRL_ADMA32;
3194         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3195
3196         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3197                      SDHCI_BLOCK_SIZE);
3198
3199         /* Set maximum timeout */
3200         sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
3201
3202         host->ier = host->cqe_ier;
3203
3204         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3205         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3206
3207         host->cqe_on = true;
3208
3209         pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3210                  mmc_hostname(mmc), host->ier,
3211                  sdhci_readl(host, SDHCI_INT_STATUS));
3212
3213         mmiowb();
3214         spin_unlock_irqrestore(&host->lock, flags);
3215 }
3216 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3217
3218 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3219 {
3220         struct sdhci_host *host = mmc_priv(mmc);
3221         unsigned long flags;
3222
3223         spin_lock_irqsave(&host->lock, flags);
3224
3225         sdhci_set_default_irqs(host);
3226
3227         host->cqe_on = false;
3228
3229         if (recovery) {
3230                 sdhci_do_reset(host, SDHCI_RESET_CMD);
3231                 sdhci_do_reset(host, SDHCI_RESET_DATA);
3232         }
3233
3234         pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3235                  mmc_hostname(mmc), host->ier,
3236                  sdhci_readl(host, SDHCI_INT_STATUS));
3237
3238         mmiowb();
3239         spin_unlock_irqrestore(&host->lock, flags);
3240 }
3241 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3242
3243 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3244                    int *data_error)
3245 {
3246         u32 mask;
3247
3248         if (!host->cqe_on)
3249                 return false;
3250
3251         if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3252                 *cmd_error = -EILSEQ;
3253         else if (intmask & SDHCI_INT_TIMEOUT)
3254                 *cmd_error = -ETIMEDOUT;
3255         else
3256                 *cmd_error = 0;
3257
3258         if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3259                 *data_error = -EILSEQ;
3260         else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3261                 *data_error = -ETIMEDOUT;
3262         else if (intmask & SDHCI_INT_ADMA_ERROR)
3263                 *data_error = -EIO;
3264         else
3265                 *data_error = 0;
3266
3267         /* Clear selected interrupts. */
3268         mask = intmask & host->cqe_ier;
3269         sdhci_writel(host, mask, SDHCI_INT_STATUS);
3270
3271         if (intmask & SDHCI_INT_BUS_POWER)
3272                 pr_err("%s: Card is consuming too much power!\n",
3273                        mmc_hostname(host->mmc));
3274
3275         intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3276         if (intmask) {
3277                 sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3278                 pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3279                        mmc_hostname(host->mmc), intmask);
3280                 sdhci_dumpregs(host);
3281         }
3282
3283         return true;
3284 }
3285 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3286
3287 /*****************************************************************************\
3288  *                                                                           *
3289  * Device allocation/registration                                            *
3290  *                                                                           *
3291 \*****************************************************************************/
3292
3293 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3294         size_t priv_size)
3295 {
3296         struct mmc_host *mmc;
3297         struct sdhci_host *host;
3298
3299         WARN_ON(dev == NULL);
3300
3301         mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3302         if (!mmc)
3303                 return ERR_PTR(-ENOMEM);
3304
3305         host = mmc_priv(mmc);
3306         host->mmc = mmc;
3307         host->mmc_host_ops = sdhci_ops;
3308         mmc->ops = &host->mmc_host_ops;
3309
3310         host->flags = SDHCI_SIGNALING_330;
3311
3312         host->cqe_ier     = SDHCI_CQE_INT_MASK;
3313         host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3314
3315         host->tuning_delay = -1;
3316
3317         host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3318
3319         return host;
3320 }
3321
3322 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3323
3324 static int sdhci_set_dma_mask(struct sdhci_host *host)
3325 {
3326         struct mmc_host *mmc = host->mmc;
3327         struct device *dev = mmc_dev(mmc);
3328         int ret = -EINVAL;
3329
3330         if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3331                 host->flags &= ~SDHCI_USE_64_BIT_DMA;
3332
3333         /* Try 64-bit mask if hardware is capable  of it */
3334         if (host->flags & SDHCI_USE_64_BIT_DMA) {
3335                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3336                 if (ret) {
3337                         pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3338                                 mmc_hostname(mmc));
3339                         host->flags &= ~SDHCI_USE_64_BIT_DMA;
3340                 }
3341         }
3342
3343         /* 32-bit mask as default & fallback */
3344         if (ret) {
3345                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3346                 if (ret)
3347                         pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3348                                 mmc_hostname(mmc));
3349         }
3350
3351         return ret;
3352 }
3353
3354 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3355 {
3356         u16 v;
3357         u64 dt_caps_mask = 0;
3358         u64 dt_caps = 0;
3359
3360         if (host->read_caps)
3361                 return;
3362
3363         host->read_caps = true;
3364
3365         if (debug_quirks)
3366                 host->quirks = debug_quirks;
3367
3368         if (debug_quirks2)
3369                 host->quirks2 = debug_quirks2;
3370
3371         sdhci_do_reset(host, SDHCI_RESET_ALL);
3372
3373         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3374                              "sdhci-caps-mask", &dt_caps_mask);
3375         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3376                              "sdhci-caps", &dt_caps);
3377
3378         v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3379         host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3380
3381         if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3382                 return;
3383
3384         if (caps) {
3385                 host->caps = *caps;
3386         } else {
3387                 host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3388                 host->caps &= ~lower_32_bits(dt_caps_mask);
3389                 host->caps |= lower_32_bits(dt_caps);
3390         }
3391
3392         if (host->version < SDHCI_SPEC_300)
3393                 return;
3394
3395         if (caps1) {
3396                 host->caps1 = *caps1;
3397         } else {
3398                 host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3399                 host->caps1 &= ~upper_32_bits(dt_caps_mask);
3400                 host->caps1 |= upper_32_bits(dt_caps);
3401         }
3402 }
3403 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3404
3405 static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3406 {
3407         struct mmc_host *mmc = host->mmc;
3408         unsigned int max_blocks;
3409         unsigned int bounce_size;
3410         int ret;
3411
3412         /*
3413          * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3414          * has diminishing returns, this is probably because SD/MMC
3415          * cards are usually optimized to handle this size of requests.
3416          */
3417         bounce_size = SZ_64K;
3418         /*
3419          * Adjust downwards to maximum request size if this is less
3420          * than our segment size, else hammer down the maximum
3421          * request size to the maximum buffer size.
3422          */
3423         if (mmc->max_req_size < bounce_size)
3424                 bounce_size = mmc->max_req_size;
3425         max_blocks = bounce_size / 512;
3426
3427         /*
3428          * When we just support one segment, we can get significant
3429          * speedups by the help of a bounce buffer to group scattered
3430          * reads/writes together.
3431          */
3432         host->bounce_buffer = devm_kmalloc(mmc->parent,
3433                                            bounce_size,
3434                                            GFP_KERNEL);
3435         if (!host->bounce_buffer) {
3436                 pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3437                        mmc_hostname(mmc),
3438                        bounce_size);
3439                 /*
3440                  * Exiting with zero here makes sure we proceed with
3441                  * mmc->max_segs == 1.
3442                  */
3443                 return 0;
3444         }
3445
3446         host->bounce_addr = dma_map_single(mmc->parent,
3447                                            host->bounce_buffer,
3448                                            bounce_size,
3449                                            DMA_BIDIRECTIONAL);
3450         ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3451         if (ret)
3452                 /* Again fall back to max_segs == 1 */
3453                 return 0;
3454         host->bounce_buffer_size = bounce_size;
3455
3456         /* Lie about this since we're bouncing */
3457         mmc->max_segs = max_blocks;
3458         mmc->max_seg_size = bounce_size;
3459         mmc->max_req_size = bounce_size;
3460
3461         pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3462                 mmc_hostname(mmc), max_blocks, bounce_size);
3463
3464         return 0;
3465 }
3466
3467 int sdhci_setup_host(struct sdhci_host *host)
3468 {
3469         struct mmc_host *mmc;
3470         u32 max_current_caps;
3471         unsigned int ocr_avail;
3472         unsigned int override_timeout_clk;
3473         u32 max_clk;
3474         int ret;
3475
3476         WARN_ON(host == NULL);
3477         if (host == NULL)
3478                 return -EINVAL;
3479
3480         mmc = host->mmc;
3481
3482         /*
3483          * If there are external regulators, get them. Note this must be done
3484          * early before resetting the host and reading the capabilities so that
3485          * the host can take the appropriate action if regulators are not
3486          * available.
3487          */
3488         ret = mmc_regulator_get_supply(mmc);
3489         if (ret)
3490                 return ret;
3491
3492         DBG("Version:   0x%08x | Present:  0x%08x\n",
3493             sdhci_readw(host, SDHCI_HOST_VERSION),
3494             sdhci_readl(host, SDHCI_PRESENT_STATE));
3495         DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3496             sdhci_readl(host, SDHCI_CAPABILITIES),
3497             sdhci_readl(host, SDHCI_CAPABILITIES_1));
3498
3499         sdhci_read_caps(host);
3500
3501         override_timeout_clk = host->timeout_clk;
3502
3503         if (host->version > SDHCI_SPEC_300) {
3504                 pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3505                        mmc_hostname(mmc), host->version);
3506         }
3507
3508         if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3509                 host->flags |= SDHCI_USE_SDMA;
3510         else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3511                 DBG("Controller doesn't have SDMA capability\n");
3512         else
3513                 host->flags |= SDHCI_USE_SDMA;
3514
3515         if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3516                 (host->flags & SDHCI_USE_SDMA)) {
3517                 DBG("Disabling DMA as it is marked broken\n");
3518                 host->flags &= ~SDHCI_USE_SDMA;
3519         }
3520
3521         if ((host->version >= SDHCI_SPEC_200) &&
3522                 (host->caps & SDHCI_CAN_DO_ADMA2))
3523                 host->flags |= SDHCI_USE_ADMA;
3524
3525         if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3526                 (host->flags & SDHCI_USE_ADMA)) {
3527                 DBG("Disabling ADMA as it is marked broken\n");
3528                 host->flags &= ~SDHCI_USE_ADMA;
3529         }
3530
3531         /*
3532          * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3533          * and *must* do 64-bit DMA.  A driver has the opportunity to change
3534          * that during the first call to ->enable_dma().  Similarly
3535          * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3536          * implement.
3537          */
3538         if (host->caps & SDHCI_CAN_64BIT)
3539                 host->flags |= SDHCI_USE_64_BIT_DMA;
3540
3541         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3542                 ret = sdhci_set_dma_mask(host);
3543
3544                 if (!ret && host->ops->enable_dma)
3545                         ret = host->ops->enable_dma(host);
3546
3547                 if (ret) {
3548                         pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3549                                 mmc_hostname(mmc));
3550                         host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3551
3552                         ret = 0;
3553                 }
3554         }
3555
3556         /* SDMA does not support 64-bit DMA */
3557         if (host->flags & SDHCI_USE_64_BIT_DMA)
3558                 host->flags &= ~SDHCI_USE_SDMA;
3559
3560         if (host->flags & SDHCI_USE_ADMA) {
3561                 dma_addr_t dma;
3562                 void *buf;
3563
3564                 /*
3565                  * The DMA descriptor table size is calculated as the maximum
3566                  * number of segments times 2, to allow for an alignment
3567                  * descriptor for each segment, plus 1 for a nop end descriptor,
3568                  * all multipled by the descriptor size.
3569                  */
3570                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
3571                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3572                                               SDHCI_ADMA2_64_DESC_SZ;
3573                         host->desc_sz = SDHCI_ADMA2_64_DESC_SZ;
3574                 } else {
3575                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3576                                               SDHCI_ADMA2_32_DESC_SZ;
3577                         host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3578                 }
3579
3580                 host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3581                 buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz +
3582                                          host->adma_table_sz, &dma, GFP_KERNEL);
3583                 if (!buf) {
3584                         pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3585                                 mmc_hostname(mmc));
3586                         host->flags &= ~SDHCI_USE_ADMA;
3587                 } else if ((dma + host->align_buffer_sz) &
3588                            (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3589                         pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3590                                 mmc_hostname(mmc));
3591                         host->flags &= ~SDHCI_USE_ADMA;
3592                         dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3593                                           host->adma_table_sz, buf, dma);
3594                 } else {
3595                         host->align_buffer = buf;
3596                         host->align_addr = dma;
3597
3598                         host->adma_table = buf + host->align_buffer_sz;
3599                         host->adma_addr = dma + host->align_buffer_sz;
3600                 }
3601         }
3602
3603         /*
3604          * If we use DMA, then it's up to the caller to set the DMA
3605          * mask, but PIO does not need the hw shim so we set a new
3606          * mask here in that case.
3607          */
3608         if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3609                 host->dma_mask = DMA_BIT_MASK(64);
3610                 mmc_dev(mmc)->dma_mask = &host->dma_mask;
3611         }
3612
3613         if (host->version >= SDHCI_SPEC_300)
3614                 host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3615                         >> SDHCI_CLOCK_BASE_SHIFT;
3616         else
3617                 host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3618                         >> SDHCI_CLOCK_BASE_SHIFT;
3619
3620         host->max_clk *= 1000000;
3621         if (host->max_clk == 0 || host->quirks &
3622                         SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3623                 if (!host->ops->get_max_clock) {
3624                         pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3625                                mmc_hostname(mmc));
3626                         ret = -ENODEV;
3627                         goto undma;
3628                 }
3629                 host->max_clk = host->ops->get_max_clock(host);
3630         }
3631
3632         /*
3633          * In case of Host Controller v3.00, find out whether clock
3634          * multiplier is supported.
3635          */
3636         host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3637                         SDHCI_CLOCK_MUL_SHIFT;
3638
3639         /*
3640          * In case the value in Clock Multiplier is 0, then programmable
3641          * clock mode is not supported, otherwise the actual clock
3642          * multiplier is one more than the value of Clock Multiplier
3643          * in the Capabilities Register.
3644          */
3645         if (host->clk_mul)
3646                 host->clk_mul += 1;
3647
3648         /*
3649          * Set host parameters.
3650          */
3651         max_clk = host->max_clk;
3652
3653         if (host->ops->get_min_clock)
3654                 mmc->f_min = host->ops->get_min_clock(host);
3655         else if (host->version >= SDHCI_SPEC_300) {
3656                 if (host->clk_mul) {
3657                         mmc->f_min = (host->max_clk * host->clk_mul) / 1024;
3658                         max_clk = host->max_clk * host->clk_mul;
3659                 } else
3660                         mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3661         } else
3662                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3663
3664         if (!mmc->f_max || mmc->f_max > max_clk)
3665                 mmc->f_max = max_clk;
3666
3667         if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3668                 host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3669                                         SDHCI_TIMEOUT_CLK_SHIFT;
3670
3671                 if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3672                         host->timeout_clk *= 1000;
3673
3674                 if (host->timeout_clk == 0) {
3675                         if (!host->ops->get_timeout_clock) {
3676                                 pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3677                                         mmc_hostname(mmc));
3678                                 ret = -ENODEV;
3679                                 goto undma;
3680                         }
3681
3682                         host->timeout_clk =
3683                                 DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3684                                              1000);
3685                 }
3686
3687                 if (override_timeout_clk)
3688                         host->timeout_clk = override_timeout_clk;
3689
3690                 mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3691                         host->ops->get_max_timeout_count(host) : 1 << 27;
3692                 mmc->max_busy_timeout /= host->timeout_clk;
3693         }
3694
3695         if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
3696             !host->ops->get_max_timeout_count)
3697                 mmc->max_busy_timeout = 0;
3698
3699         mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3700         mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3701
3702         if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3703                 host->flags |= SDHCI_AUTO_CMD12;
3704
3705         /* Auto-CMD23 stuff only works in ADMA or PIO. */
3706         if ((host->version >= SDHCI_SPEC_300) &&
3707             ((host->flags & SDHCI_USE_ADMA) ||
3708              !(host->flags & SDHCI_USE_SDMA)) &&
3709              !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3710                 host->flags |= SDHCI_AUTO_CMD23;
3711                 DBG("Auto-CMD23 available\n");
3712         } else {
3713                 DBG("Auto-CMD23 unavailable\n");
3714         }
3715
3716         /*
3717          * A controller may support 8-bit width, but the board itself
3718          * might not have the pins brought out.  Boards that support
3719          * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3720          * their platform code before calling sdhci_add_host(), and we
3721          * won't assume 8-bit width for hosts without that CAP.
3722          */
3723         if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3724                 mmc->caps |= MMC_CAP_4_BIT_DATA;
3725
3726         if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3727                 mmc->caps &= ~MMC_CAP_CMD23;
3728
3729         if (host->caps & SDHCI_CAN_DO_HISPD)
3730                 mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3731
3732         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3733             mmc_card_is_removable(mmc) &&
3734             mmc_gpio_get_cd(host->mmc) < 0)
3735                 mmc->caps |= MMC_CAP_NEEDS_POLL;
3736
3737         /* If vqmmc regulator and no 1.8V signalling, then there's no UHS */
3738         if (!IS_ERR(mmc->supply.vqmmc)) {
3739                 ret = regulator_enable(mmc->supply.vqmmc);
3740                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3741                                                     1950000))
3742                         host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3743                                          SDHCI_SUPPORT_SDR50 |
3744                                          SDHCI_SUPPORT_DDR50);
3745                 if (ret) {
3746                         pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3747                                 mmc_hostname(mmc), ret);
3748                         mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3749                 }
3750         }
3751
3752         if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3753                 host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3754                                  SDHCI_SUPPORT_DDR50);
3755                 /*
3756                  * The SDHCI controller in a SoC might support HS200/HS400
3757                  * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
3758                  * but if the board is modeled such that the IO lines are not
3759                  * connected to 1.8v then HS200/HS400 cannot be supported.
3760                  * Disable HS200/HS400 if the board does not have 1.8v connected
3761                  * to the IO lines. (Applicable for other modes in 1.8v)
3762                  */
3763                 mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
3764                 mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
3765         }
3766
3767         /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3768         if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3769                            SDHCI_SUPPORT_DDR50))
3770                 mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3771
3772         /* SDR104 supports also implies SDR50 support */
3773         if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3774                 mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3775                 /* SD3.0: SDR104 is supported so (for eMMC) the caps2
3776                  * field can be promoted to support HS200.
3777                  */
3778                 if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3779                         mmc->caps2 |= MMC_CAP2_HS200;
3780         } else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3781                 mmc->caps |= MMC_CAP_UHS_SDR50;
3782         }
3783
3784         if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3785             (host->caps1 & SDHCI_SUPPORT_HS400))
3786                 mmc->caps2 |= MMC_CAP2_HS400;
3787
3788         if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3789             (IS_ERR(mmc->supply.vqmmc) ||
3790              !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3791                                              1300000)))
3792                 mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3793
3794         if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3795             !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3796                 mmc->caps |= MMC_CAP_UHS_DDR50;
3797
3798         /* Does the host need tuning for SDR50? */
3799         if (host->caps1 & SDHCI_USE_SDR50_TUNING)
3800                 host->flags |= SDHCI_SDR50_NEEDS_TUNING;
3801
3802         /* Driver Type(s) (A, C, D) supported by the host */
3803         if (host->caps1 & SDHCI_DRIVER_TYPE_A)
3804                 mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
3805         if (host->caps1 & SDHCI_DRIVER_TYPE_C)
3806                 mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
3807         if (host->caps1 & SDHCI_DRIVER_TYPE_D)
3808                 mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
3809
3810         /* Initial value for re-tuning timer count */
3811         host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
3812                              SDHCI_RETUNING_TIMER_COUNT_SHIFT;
3813
3814         /*
3815          * In case Re-tuning Timer is not disabled, the actual value of
3816          * re-tuning timer will be 2 ^ (n - 1).
3817          */
3818         if (host->tuning_count)
3819                 host->tuning_count = 1 << (host->tuning_count - 1);
3820
3821         /* Re-tuning mode supported by the Host Controller */
3822         host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
3823                              SDHCI_RETUNING_MODE_SHIFT;
3824
3825         ocr_avail = 0;
3826
3827         /*
3828          * According to SD Host Controller spec v3.00, if the Host System
3829          * can afford more than 150mA, Host Driver should set XPC to 1. Also
3830          * the value is meaningful only if Voltage Support in the Capabilities
3831          * register is set. The actual current value is 4 times the register
3832          * value.
3833          */
3834         max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
3835         if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
3836                 int curr = regulator_get_current_limit(mmc->supply.vmmc);
3837                 if (curr > 0) {
3838
3839                         /* convert to SDHCI_MAX_CURRENT format */
3840                         curr = curr/1000;  /* convert to mA */
3841                         curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
3842
3843                         curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
3844                         max_current_caps =
3845                                 (curr << SDHCI_MAX_CURRENT_330_SHIFT) |
3846                                 (curr << SDHCI_MAX_CURRENT_300_SHIFT) |
3847                                 (curr << SDHCI_MAX_CURRENT_180_SHIFT);
3848                 }
3849         }
3850
3851         if (host->caps & SDHCI_CAN_VDD_330) {
3852                 ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
3853
3854                 mmc->max_current_330 = ((max_current_caps &
3855                                    SDHCI_MAX_CURRENT_330_MASK) >>
3856                                    SDHCI_MAX_CURRENT_330_SHIFT) *
3857                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3858         }
3859         if (host->caps & SDHCI_CAN_VDD_300) {
3860                 ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
3861
3862                 mmc->max_current_300 = ((max_current_caps &
3863                                    SDHCI_MAX_CURRENT_300_MASK) >>
3864                                    SDHCI_MAX_CURRENT_300_SHIFT) *
3865                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3866         }
3867         if (host->caps & SDHCI_CAN_VDD_180) {
3868                 ocr_avail |= MMC_VDD_165_195;
3869
3870                 mmc->max_current_180 = ((max_current_caps &
3871                                    SDHCI_MAX_CURRENT_180_MASK) >>
3872                                    SDHCI_MAX_CURRENT_180_SHIFT) *
3873                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3874         }
3875
3876         /* If OCR set by host, use it instead. */
3877         if (host->ocr_mask)
3878                 ocr_avail = host->ocr_mask;
3879
3880         /* If OCR set by external regulators, give it highest prio. */
3881         if (mmc->ocr_avail)
3882                 ocr_avail = mmc->ocr_avail;
3883
3884         mmc->ocr_avail = ocr_avail;
3885         mmc->ocr_avail_sdio = ocr_avail;
3886         if (host->ocr_avail_sdio)
3887                 mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
3888         mmc->ocr_avail_sd = ocr_avail;
3889         if (host->ocr_avail_sd)
3890                 mmc->ocr_avail_sd &= host->ocr_avail_sd;
3891         else /* normal SD controllers don't support 1.8V */
3892                 mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
3893         mmc->ocr_avail_mmc = ocr_avail;
3894         if (host->ocr_avail_mmc)
3895                 mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
3896
3897         if (mmc->ocr_avail == 0) {
3898                 pr_err("%s: Hardware doesn't report any support voltages.\n",
3899                        mmc_hostname(mmc));
3900                 ret = -ENODEV;
3901                 goto unreg;
3902         }
3903
3904         if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
3905                           MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
3906                           MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
3907             (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
3908                 host->flags |= SDHCI_SIGNALING_180;
3909
3910         if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
3911                 host->flags |= SDHCI_SIGNALING_120;
3912
3913         spin_lock_init(&host->lock);
3914
3915         /*
3916          * Maximum number of sectors in one transfer. Limited by SDMA boundary
3917          * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
3918          * is less anyway.
3919          */
3920         mmc->max_req_size = 524288;
3921
3922         /*
3923          * Maximum number of segments. Depends on if the hardware
3924          * can do scatter/gather or not.
3925          */
3926         if (host->flags & SDHCI_USE_ADMA) {
3927                 mmc->max_segs = SDHCI_MAX_SEGS;
3928         } else if (host->flags & SDHCI_USE_SDMA) {
3929                 mmc->max_segs = 1;
3930                 if (swiotlb_max_segment()) {
3931                         unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
3932                                                 IO_TLB_SEGSIZE;
3933                         mmc->max_req_size = min(mmc->max_req_size,
3934                                                 max_req_size);
3935                 }
3936         } else { /* PIO */
3937                 mmc->max_segs = SDHCI_MAX_SEGS;
3938         }
3939
3940         /*
3941          * Maximum segment size. Could be one segment with the maximum number
3942          * of bytes. When doing hardware scatter/gather, each entry cannot
3943          * be larger than 64 KiB though.
3944          */
3945         if (host->flags & SDHCI_USE_ADMA) {
3946                 if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
3947                         mmc->max_seg_size = 65535;
3948                 else
3949                         mmc->max_seg_size = 65536;
3950         } else {
3951                 mmc->max_seg_size = mmc->max_req_size;
3952         }
3953
3954         /*
3955          * Maximum block size. This varies from controller to controller and
3956          * is specified in the capabilities register.
3957          */
3958         if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
3959                 mmc->max_blk_size = 2;
3960         } else {
3961                 mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
3962                                 SDHCI_MAX_BLOCK_SHIFT;
3963                 if (mmc->max_blk_size >= 3) {
3964                         pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
3965                                 mmc_hostname(mmc));
3966                         mmc->max_blk_size = 0;
3967                 }
3968         }
3969
3970         mmc->max_blk_size = 512 << mmc->max_blk_size;
3971
3972         /*
3973          * Maximum block count.
3974          */
3975         mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
3976
3977         if (mmc->max_segs == 1) {
3978                 /* This may alter mmc->*_blk_* parameters */
3979                 ret = sdhci_allocate_bounce_buffer(host);
3980                 if (ret)
3981                         return ret;
3982         }
3983
3984         return 0;
3985
3986 unreg:
3987         if (!IS_ERR(mmc->supply.vqmmc))
3988                 regulator_disable(mmc->supply.vqmmc);
3989 undma:
3990         if (host->align_buffer)
3991                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3992                                   host->adma_table_sz, host->align_buffer,
3993                                   host->align_addr);
3994         host->adma_table = NULL;
3995         host->align_buffer = NULL;
3996
3997         return ret;
3998 }
3999 EXPORT_SYMBOL_GPL(sdhci_setup_host);
4000
4001 void sdhci_cleanup_host(struct sdhci_host *host)
4002 {
4003         struct mmc_host *mmc = host->mmc;
4004
4005         if (!IS_ERR(mmc->supply.vqmmc))
4006                 regulator_disable(mmc->supply.vqmmc);
4007
4008         if (host->align_buffer)
4009                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4010                                   host->adma_table_sz, host->align_buffer,
4011                                   host->align_addr);
4012         host->adma_table = NULL;
4013         host->align_buffer = NULL;
4014 }
4015 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
4016
4017 int __sdhci_add_host(struct sdhci_host *host)
4018 {
4019         struct mmc_host *mmc = host->mmc;
4020         int ret;
4021
4022         /*
4023          * Init tasklets.
4024          */
4025         tasklet_init(&host->finish_tasklet,
4026                 sdhci_tasklet_finish, (unsigned long)host);
4027
4028         timer_setup(&host->timer, sdhci_timeout_timer, 0);
4029         timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);
4030
4031         init_waitqueue_head(&host->buf_ready_int);
4032
4033         sdhci_init(host, 0);
4034
4035         ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
4036                                    IRQF_SHARED, mmc_hostname(mmc), host);
4037         if (ret) {
4038                 pr_err("%s: Failed to request IRQ %d: %d\n",
4039                        mmc_hostname(mmc), host->irq, ret);
4040                 goto untasklet;
4041         }
4042
4043         ret = sdhci_led_register(host);
4044         if (ret) {
4045                 pr_err("%s: Failed to register LED device: %d\n",
4046                        mmc_hostname(mmc), ret);
4047                 goto unirq;
4048         }
4049
4050         mmiowb();
4051
4052         ret = mmc_add_host(mmc);
4053         if (ret)
4054                 goto unled;
4055
4056         pr_info("%s: SDHCI controller on %s [%s] using %s\n",
4057                 mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
4058                 (host->flags & SDHCI_USE_ADMA) ?
4059                 (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
4060                 (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4061
4062         sdhci_enable_card_detection(host);
4063
4064         return 0;
4065
4066 unled:
4067         sdhci_led_unregister(host);
4068 unirq:
4069         sdhci_do_reset(host, SDHCI_RESET_ALL);
4070         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4071         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4072         free_irq(host->irq, host);
4073 untasklet:
4074         tasklet_kill(&host->finish_tasklet);
4075
4076         return ret;
4077 }
4078 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4079
4080 int sdhci_add_host(struct sdhci_host *host)
4081 {
4082         int ret;
4083
4084         ret = sdhci_setup_host(host);
4085         if (ret)
4086                 return ret;
4087
4088         ret = __sdhci_add_host(host);
4089         if (ret)
4090                 goto cleanup;
4091
4092         return 0;
4093
4094 cleanup:
4095         sdhci_cleanup_host(host);
4096
4097         return ret;
4098 }
4099 EXPORT_SYMBOL_GPL(sdhci_add_host);
4100
4101 void sdhci_remove_host(struct sdhci_host *host, int dead)
4102 {
4103         struct mmc_host *mmc = host->mmc;
4104         unsigned long flags;
4105
4106         if (dead) {
4107                 spin_lock_irqsave(&host->lock, flags);
4108
4109                 host->flags |= SDHCI_DEVICE_DEAD;
4110
4111                 if (sdhci_has_requests(host)) {
4112                         pr_err("%s: Controller removed during "
4113                                 " transfer!\n", mmc_hostname(mmc));
4114                         sdhci_error_out_mrqs(host, -ENOMEDIUM);
4115                 }
4116
4117                 spin_unlock_irqrestore(&host->lock, flags);
4118         }
4119
4120         sdhci_disable_card_detection(host);
4121
4122         mmc_remove_host(mmc);
4123
4124         sdhci_led_unregister(host);
4125
4126         if (!dead)
4127                 sdhci_do_reset(host, SDHCI_RESET_ALL);
4128
4129         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4130         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4131         free_irq(host->irq, host);
4132
4133         del_timer_sync(&host->timer);
4134         del_timer_sync(&host->data_timer);
4135
4136         tasklet_kill(&host->finish_tasklet);
4137
4138         if (!IS_ERR(mmc->supply.vqmmc))
4139                 regulator_disable(mmc->supply.vqmmc);
4140
4141         if (host->align_buffer)
4142                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4143                                   host->adma_table_sz, host->align_buffer,
4144                                   host->align_addr);
4145
4146         host->adma_table = NULL;
4147         host->align_buffer = NULL;
4148 }
4149
4150 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4151
4152 void sdhci_free_host(struct sdhci_host *host)
4153 {
4154         mmc_free_host(host->mmc);
4155 }
4156
4157 EXPORT_SYMBOL_GPL(sdhci_free_host);
4158
4159 /*****************************************************************************\
4160  *                                                                           *
4161  * Driver init/exit                                                          *
4162  *                                                                           *
4163 \*****************************************************************************/
4164
4165 static int __init sdhci_drv_init(void)
4166 {
4167         pr_info(DRIVER_NAME
4168                 ": Secure Digital Host Controller Interface driver\n");
4169         pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4170
4171         return 0;
4172 }
4173
4174 static void __exit sdhci_drv_exit(void)
4175 {
4176 }
4177
4178 module_init(sdhci_drv_init);
4179 module_exit(sdhci_drv_exit);
4180
4181 module_param(debug_quirks, uint, 0444);
4182 module_param(debug_quirks2, uint, 0444);
4183
4184 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4185 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4186 MODULE_LICENSE("GPL");
4187
4188 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4189 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");