drivers/mmc/host/mmci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/drivers/mmc/host/mmci.c - ARM PrimeCell MMCI PL180/1 driver
   4  *
   5  *  Copyright (C) 2003 Deep Blue Solutions, Ltd, All Rights Reserved.
   6  *  Copyright (C) 2010 ST-Ericsson SA
   7  */
   8 #include <linux/module.h>
   9 #include <linux/moduleparam.h>
  10 #include <linux/init.h>
  11 #include <linux/ioport.h>
  12 #include <linux/device.h>
  13 #include <linux/io.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/kernel.h>
  16 #include <linux/slab.h>
  17 #include <linux/delay.h>
  18 #include <linux/err.h>
  19 #include <linux/highmem.h>
  20 #include <linux/log2.h>
  21 #include <linux/mmc/mmc.h>
  22 #include <linux/mmc/pm.h>
  23 #include <linux/mmc/host.h>
  24 #include <linux/mmc/card.h>
  25 #include <linux/mmc/sd.h>
  26 #include <linux/mmc/slot-gpio.h>
  27 #include <linux/amba/bus.h>
  28 #include <linux/clk.h>
  29 #include <linux/scatterlist.h>
  30 #include <linux/of.h>
  31 #include <linux/regulator/consumer.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/amba/mmci.h>
  35 #include <linux/pm_runtime.h>
  36 #include <linux/types.h>
  37 #include <linux/pinctrl/consumer.h>
  38 #include <linux/reset.h>
  39 #include <linux/gpio/consumer.h>
  40 #include <linux/workqueue.h>
  41
  42 #include <asm/div64.h>
  43 #include <asm/io.h>
  44
  45 #include "mmci.h"
  46
  47 #define DRIVER_NAME "mmci-pl18x"
  48
  49 static void mmci_variant_init(struct mmci_host *host);
  50 static void ux500_variant_init(struct mmci_host *host);
  51 static void ux500v2_variant_init(struct mmci_host *host);
  52
  53 static unsigned int fmax = 515633;
  54
  55 static struct variant_data variant_arm = {
  56         .fifosize               = 16 * 4,
  57         .fifohalfsize           = 8 * 4,
  58         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
  59         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
  60         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
  61         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
  62         .datalength_bits        = 16,
  63         .datactrl_blocksz       = 11,
  64         .pwrreg_powerup         = MCI_PWR_UP,
  65         .f_max                  = 100000000,
  66         .reversed_irq_handling  = true,
  67         .mmcimask1              = true,
  68         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
  69         .start_err              = MCI_STARTBITERR,
  70         .opendrain              = MCI_ROD,
  71         .init                   = mmci_variant_init,
  72 };
  73
  74 static struct variant_data variant_arm_extended_fifo = {
  75         .fifosize               = 128 * 4,
  76         .fifohalfsize           = 64 * 4,
  77         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
  78         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
  79         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
  80         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
  81         .datalength_bits        = 16,
  82         .datactrl_blocksz       = 11,
  83         .pwrreg_powerup         = MCI_PWR_UP,
  84         .f_max                  = 100000000,
  85         .mmcimask1              = true,
  86         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
  87         .start_err              = MCI_STARTBITERR,
  88         .opendrain              = MCI_ROD,
  89         .init                   = mmci_variant_init,
  90 };
  91
  92 static struct variant_data variant_arm_extended_fifo_hwfc = {
  93         .fifosize               = 128 * 4,
  94         .fifohalfsize           = 64 * 4,
  95         .clkreg_enable          = MCI_ARM_HWFCEN,
  96         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
  97         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
  98         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
  99         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 100         .datalength_bits        = 16,
 101         .datactrl_blocksz       = 11,
 102         .pwrreg_powerup         = MCI_PWR_UP,
 103         .f_max                  = 100000000,
 104         .mmcimask1              = true,
 105         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 106         .start_err              = MCI_STARTBITERR,
 107         .opendrain              = MCI_ROD,
 108         .init                   = mmci_variant_init,
 109 };
 110
 111 static struct variant_data variant_u300 = {
 112         .fifosize               = 16 * 4,
 113         .fifohalfsize           = 8 * 4,
 114         .clkreg_enable          = MCI_ST_U300_HWFCEN,
 115         .clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 116         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 117         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 118         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 119         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 120         .datalength_bits        = 16,
 121         .datactrl_blocksz       = 11,
 122         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 123         .st_sdio                        = true,
 124         .pwrreg_powerup         = MCI_PWR_ON,
 125         .f_max                  = 100000000,
 126         .signal_direction       = true,
 127         .pwrreg_clkgate         = true,
 128         .pwrreg_nopower         = true,
 129         .mmcimask1              = true,
 130         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 131         .start_err              = MCI_STARTBITERR,
 132         .opendrain              = MCI_OD,
 133         .init                   = mmci_variant_init,
 134 };
 135
 136 static struct variant_data variant_nomadik = {
 137         .fifosize               = 16 * 4,
 138         .fifohalfsize           = 8 * 4,
 139         .clkreg                 = MCI_CLK_ENABLE,
 140         .clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 141         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 142         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 143         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 144         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 145         .datalength_bits        = 24,
 146         .datactrl_blocksz       = 11,
 147         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 148         .st_sdio                = true,
 149         .st_clkdiv              = true,
 150         .pwrreg_powerup         = MCI_PWR_ON,
 151         .f_max                  = 100000000,
 152         .signal_direction       = true,
 153         .pwrreg_clkgate         = true,
 154         .pwrreg_nopower         = true,
 155         .mmcimask1              = true,
 156         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 157         .start_err              = MCI_STARTBITERR,
 158         .opendrain              = MCI_OD,
 159         .init                   = mmci_variant_init,
 160 };
 161
 162 static struct variant_data variant_ux500 = {
 163         .fifosize               = 30 * 4,
 164         .fifohalfsize           = 8 * 4,
 165         .clkreg                 = MCI_CLK_ENABLE,
 166         .clkreg_enable          = MCI_ST_UX500_HWFCEN,
 167         .clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 168         .clkreg_neg_edge_enable = MCI_ST_UX500_NEG_EDGE,
 169         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 170         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 171         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 172         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 173         .datalength_bits        = 24,
 174         .datactrl_blocksz       = 11,
 175         .datactrl_any_blocksz   = true,
 176         .dma_power_of_2         = true,
 177         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 178         .st_sdio                = true,
 179         .st_clkdiv              = true,
 180         .pwrreg_powerup         = MCI_PWR_ON,
 181         .f_max                  = 100000000,
 182         .signal_direction       = true,
 183         .pwrreg_clkgate         = true,
 184         .busy_detect            = true,
 185         .busy_dpsm_flag         = MCI_DPSM_ST_BUSYMODE,
 186         .busy_detect_flag       = MCI_ST_CARDBUSY,
 187         .busy_detect_mask       = MCI_ST_BUSYENDMASK,
 188         .pwrreg_nopower         = true,
 189         .mmcimask1              = true,
 190         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 191         .start_err              = MCI_STARTBITERR,
 192         .opendrain              = MCI_OD,
 193         .init                   = ux500_variant_init,
 194 };
 195
 196 static struct variant_data variant_ux500v2 = {
 197         .fifosize               = 30 * 4,
 198         .fifohalfsize           = 8 * 4,
 199         .clkreg                 = MCI_CLK_ENABLE,
 200         .clkreg_enable          = MCI_ST_UX500_HWFCEN,
 201         .clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 202         .clkreg_neg_edge_enable = MCI_ST_UX500_NEG_EDGE,
 203         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 204         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 205         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 206         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 207         .datactrl_mask_ddrmode  = MCI_DPSM_ST_DDRMODE,
 208         .datalength_bits        = 24,
 209         .datactrl_blocksz       = 11,
 210         .datactrl_any_blocksz   = true,
 211         .dma_power_of_2         = true,
 212         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 213         .st_sdio                = true,
 214         .st_clkdiv              = true,
 215         .pwrreg_powerup         = MCI_PWR_ON,
 216         .f_max                  = 100000000,
 217         .signal_direction       = true,
 218         .pwrreg_clkgate         = true,
 219         .busy_detect            = true,
 220         .busy_dpsm_flag         = MCI_DPSM_ST_BUSYMODE,
 221         .busy_detect_flag       = MCI_ST_CARDBUSY,
 222         .busy_detect_mask       = MCI_ST_BUSYENDMASK,
 223         .pwrreg_nopower         = true,
 224         .mmcimask1              = true,
 225         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 226         .start_err              = MCI_STARTBITERR,
 227         .opendrain              = MCI_OD,
 228         .init                   = ux500v2_variant_init,
 229 };
 230
 231 static struct variant_data variant_stm32 = {
 232         .fifosize               = 32 * 4,
 233         .fifohalfsize           = 8 * 4,
 234         .clkreg                 = MCI_CLK_ENABLE,
 235         .clkreg_enable          = MCI_ST_UX500_HWFCEN,
 236         .clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 237         .clkreg_neg_edge_enable = MCI_ST_UX500_NEG_EDGE,
 238         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 239         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 240         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 241         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 242         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 243         .datalength_bits        = 24,
 244         .datactrl_blocksz       = 11,
 245         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 246         .st_sdio                = true,
 247         .st_clkdiv              = true,
 248         .pwrreg_powerup         = MCI_PWR_ON,
 249         .f_max                  = 48000000,
 250         .pwrreg_clkgate         = true,
 251         .pwrreg_nopower         = true,
 252         .init                   = mmci_variant_init,
 253 };
 254
 255 static struct variant_data variant_stm32_sdmmc = {
 256         .fifosize               = 16 * 4,
 257         .fifohalfsize           = 8 * 4,
 258         .f_max                  = 208000000,
 259         .stm32_clkdiv           = true,
 260         .cmdreg_cpsm_enable     = MCI_CPSM_STM32_ENABLE,
 261         .cmdreg_lrsp_crc        = MCI_CPSM_STM32_LRSP_CRC,
 262         .cmdreg_srsp_crc        = MCI_CPSM_STM32_SRSP_CRC,
 263         .cmdreg_srsp            = MCI_CPSM_STM32_SRSP,
 264         .cmdreg_stop            = MCI_CPSM_STM32_CMDSTOP,
 265         .data_cmd_enable        = MCI_CPSM_STM32_CMDTRANS,
 266         .irq_pio_mask           = MCI_IRQ_PIO_STM32_MASK,
 267         .datactrl_first         = true,
 268         .datacnt_useless        = true,
 269         .datalength_bits        = 25,
 270         .datactrl_blocksz       = 14,
 271         .datactrl_any_blocksz   = true,
 272         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 273         .stm32_idmabsize_mask   = GENMASK(12, 5),
 274         .stm32_idmabsize_align  = BIT(5),
 275         .busy_timeout           = true,
 276         .busy_detect            = true,
 277         .busy_detect_flag       = MCI_STM32_BUSYD0,
 278         .busy_detect_mask       = MCI_STM32_BUSYD0ENDMASK,
 279         .init                   = sdmmc_variant_init,
 280 };
 281
 282 static struct variant_data variant_stm32_sdmmcv2 = {
 283         .fifosize               = 16 * 4,
 284         .fifohalfsize           = 8 * 4,
 285         .f_max                  = 267000000,
 286         .stm32_clkdiv           = true,
 287         .cmdreg_cpsm_enable     = MCI_CPSM_STM32_ENABLE,
 288         .cmdreg_lrsp_crc        = MCI_CPSM_STM32_LRSP_CRC,
 289         .cmdreg_srsp_crc        = MCI_CPSM_STM32_SRSP_CRC,
 290         .cmdreg_srsp            = MCI_CPSM_STM32_SRSP,
 291         .cmdreg_stop            = MCI_CPSM_STM32_CMDSTOP,
 292         .data_cmd_enable        = MCI_CPSM_STM32_CMDTRANS,
 293         .irq_pio_mask           = MCI_IRQ_PIO_STM32_MASK,
 294         .datactrl_first         = true,
 295         .datacnt_useless        = true,
 296         .datalength_bits        = 25,
 297         .datactrl_blocksz       = 14,
 298         .datactrl_any_blocksz   = true,
 299         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 300         .stm32_idmabsize_mask   = GENMASK(16, 5),
 301         .stm32_idmabsize_align  = BIT(5),
 302         .dma_lli                = true,
 303         .busy_timeout           = true,
 304         .busy_detect            = true,
 305         .busy_detect_flag       = MCI_STM32_BUSYD0,
 306         .busy_detect_mask       = MCI_STM32_BUSYD0ENDMASK,
 307         .init                   = sdmmc_variant_init,
 308 };
 309
 310 static struct variant_data variant_stm32_sdmmcv3 = {
 311         .fifosize               = 256 * 4,
 312         .fifohalfsize           = 128 * 4,
 313         .f_max                  = 267000000,
 314         .stm32_clkdiv           = true,
 315         .cmdreg_cpsm_enable     = MCI_CPSM_STM32_ENABLE,
 316         .cmdreg_lrsp_crc        = MCI_CPSM_STM32_LRSP_CRC,
 317         .cmdreg_srsp_crc        = MCI_CPSM_STM32_SRSP_CRC,
 318         .cmdreg_srsp            = MCI_CPSM_STM32_SRSP,
 319         .cmdreg_stop            = MCI_CPSM_STM32_CMDSTOP,
 320         .data_cmd_enable        = MCI_CPSM_STM32_CMDTRANS,
 321         .irq_pio_mask           = MCI_IRQ_PIO_STM32_MASK,
 322         .datactrl_first         = true,
 323         .datacnt_useless        = true,
 324         .datalength_bits        = 25,
 325         .datactrl_blocksz       = 14,
 326         .datactrl_any_blocksz   = true,
 327         .datactrl_mask_sdio     = MCI_DPSM_ST_SDIOEN,
 328         .stm32_idmabsize_mask   = GENMASK(16, 6),
 329         .stm32_idmabsize_align  = BIT(6),
 330         .dma_lli                = true,
 331         .busy_timeout           = true,
 332         .busy_detect            = true,
 333         .busy_detect_flag       = MCI_STM32_BUSYD0,
 334         .busy_detect_mask       = MCI_STM32_BUSYD0ENDMASK,
 335         .init                   = sdmmc_variant_init,
 336 };
 337
 338 static struct variant_data variant_qcom = {
 339         .fifosize               = 16 * 4,
 340         .fifohalfsize           = 8 * 4,
 341         .clkreg                 = MCI_CLK_ENABLE,
 342         .clkreg_enable          = MCI_QCOM_CLK_FLOWENA |
 343                                   MCI_QCOM_CLK_SELECT_IN_FBCLK,
 344         .clkreg_8bit_bus_enable = MCI_QCOM_CLK_WIDEBUS_8,
 345         .datactrl_mask_ddrmode  = MCI_QCOM_CLK_SELECT_IN_DDR_MODE,
 346         .cmdreg_cpsm_enable     = MCI_CPSM_ENABLE,
 347         .cmdreg_lrsp_crc        = MCI_CPSM_RESPONSE | MCI_CPSM_LONGRSP,
 348         .cmdreg_srsp_crc        = MCI_CPSM_RESPONSE,
 349         .cmdreg_srsp            = MCI_CPSM_RESPONSE,
 350         .data_cmd_enable        = MCI_CPSM_QCOM_DATCMD,
 351         .datalength_bits        = 24,
 352         .datactrl_blocksz       = 11,
 353         .datactrl_any_blocksz   = true,
 354         .pwrreg_powerup         = MCI_PWR_UP,
 355         .f_max                  = 208000000,
 356         .explicit_mclk_control  = true,
 357         .qcom_fifo              = true,
 358         .qcom_dml               = true,
 359         .mmcimask1              = true,
 360         .irq_pio_mask           = MCI_IRQ_PIO_MASK,
 361         .start_err              = MCI_STARTBITERR,
 362         .opendrain              = MCI_ROD,
 363         .init                   = qcom_variant_init,
 364 };
 365
 366 /* Busy detection for the ST Micro variant */
 367 static int mmci_card_busy(struct mmc_host *mmc)
 368 {
 369         struct mmci_host *host = mmc_priv(mmc);
 370         unsigned long flags;
 371         int busy = 0;
 372
 373         spin_lock_irqsave(&host->lock, flags);
 374         if (readl(host->base + MMCISTATUS) & host->variant->busy_detect_flag)
 375                 busy = 1;
 376         spin_unlock_irqrestore(&host->lock, flags);
 377
 378         return busy;
 379 }
 380
 381 static void mmci_reg_delay(struct mmci_host *host)
 382 {
 383         /*
 384          * According to the spec, at least three feedback clock cycles
 385          * of max 52 MHz must pass between two writes to the MMCICLOCK reg.
 386          * Three MCLK clock cycles must pass between two MMCIPOWER reg writes.
 387          * Worst delay time during card init is at 100 kHz => 30 us.
 388          * Worst delay time when up and running is at 25 MHz => 120 ns.
 389          */
 390         if (host->cclk < 25000000)
 391                 udelay(30);
 392         else
 393                 ndelay(120);
 394 }
 395
 396 /*
 397  * This must be called with host->lock held
 398  */
 399 void mmci_write_clkreg(struct mmci_host *host, u32 clk)
 400 {
 401         if (host->clk_reg != clk) {
 402                 host->clk_reg = clk;
 403                 writel(clk, host->base + MMCICLOCK);
 404         }
 405 }
 406
 407 /*
 408  * This must be called with host->lock held
 409  */
 410 void mmci_write_pwrreg(struct mmci_host *host, u32 pwr)
 411 {
 412         if (host->pwr_reg != pwr) {
 413                 host->pwr_reg = pwr;
 414                 writel(pwr, host->base + MMCIPOWER);
 415         }
 416 }
 417
 418 /*
 419  * This must be called with host->lock held
 420  */
 421 static void mmci_write_datactrlreg(struct mmci_host *host, u32 datactrl)
 422 {
 423         /* Keep busy mode in DPSM if enabled */
 424         datactrl |= host->datactrl_reg & host->variant->busy_dpsm_flag;
 425
 426         if (host->datactrl_reg != datactrl) {
 427                 host->datactrl_reg = datactrl;
 428                 writel(datactrl, host->base + MMCIDATACTRL);
 429         }
 430 }
 431
 432 /*
 433  * This must be called with host->lock held
 434  */
 435 static void mmci_set_clkreg(struct mmci_host *host, unsigned int desired)
 436 {
 437         struct variant_data *variant = host->variant;
 438         u32 clk = variant->clkreg;
 439
 440         /* Make sure cclk reflects the current calculated clock */
 441         host->cclk = 0;
 442
 443         if (desired) {
 444                 if (variant->explicit_mclk_control) {
 445                         host->cclk = host->mclk;
 446                 } else if (desired >= host->mclk) {
 447                         clk = MCI_CLK_BYPASS;
 448                         if (variant->st_clkdiv)
 449                                 clk |= MCI_ST_UX500_NEG_EDGE;
 450                         host->cclk = host->mclk;
 451                 } else if (variant->st_clkdiv) {
 452                         /*
 453                          * DB8500 TRM says f = mclk / (clkdiv + 2)
 454                          * => clkdiv = (mclk / f) - 2
 455                          * Round the divider up so we don't exceed the max
 456                          * frequency
 457                          */
 458                         clk = DIV_ROUND_UP(host->mclk, desired) - 2;
 459                         if (clk >= 256)
 460                                 clk = 255;
 461                         host->cclk = host->mclk / (clk + 2);
 462                 } else {
 463                         /*
 464                          * PL180 TRM says f = mclk / (2 * (clkdiv + 1))
 465                          * => clkdiv = mclk / (2 * f) - 1
 466                          */
 467                         clk = host->mclk / (2 * desired) - 1;
 468                         if (clk >= 256)
 469                                 clk = 255;
 470                         host->cclk = host->mclk / (2 * (clk + 1));
 471                 }
 472
 473                 clk |= variant->clkreg_enable;
 474                 clk |= MCI_CLK_ENABLE;
 475                 /* This hasn't proven to be worthwhile */
 476                 /* clk |= MCI_CLK_PWRSAVE; */
 477         }
 478
 479         /* Set actual clock for debug */
 480         host->mmc->actual_clock = host->cclk;
 481
 482         if (host->mmc->ios.bus_width == MMC_BUS_WIDTH_4)
 483                 clk |= MCI_4BIT_BUS;
 484         if (host->mmc->ios.bus_width == MMC_BUS_WIDTH_8)
 485                 clk |= variant->clkreg_8bit_bus_enable;
 486
 487         if (host->mmc->ios.timing == MMC_TIMING_UHS_DDR50 ||
 488             host->mmc->ios.timing == MMC_TIMING_MMC_DDR52)
 489                 clk |= variant->clkreg_neg_edge_enable;
 490
 491         mmci_write_clkreg(host, clk);
 492 }
 493
 494 static void mmci_dma_release(struct mmci_host *host)
 495 {
 496         if (host->ops && host->ops->dma_release)
 497                 host->ops->dma_release(host);
 498
 499         host->use_dma = false;
 500 }
 501
 502 static void mmci_dma_setup(struct mmci_host *host)
 503 {
 504         if (!host->ops || !host->ops->dma_setup)
 505                 return;
 506
 507         if (host->ops->dma_setup(host))
 508                 return;
 509
 510         /* initialize pre request cookie */
 511         host->next_cookie = 1;
 512
 513         host->use_dma = true;
 514 }
 515
 516 /*
 517  * Validate mmc prerequisites
 518  */
 519 static int mmci_validate_data(struct mmci_host *host,
 520                               struct mmc_data *data)
 521 {
 522         struct variant_data *variant = host->variant;
 523
 524         if (!data)
 525                 return 0;
 526         if (!is_power_of_2(data->blksz) && !variant->datactrl_any_blocksz) {
 527                 dev_err(mmc_dev(host->mmc),
 528                         "unsupported block size (%d bytes)\n", data->blksz);
 529                 return -EINVAL;
 530         }
 531
 532         if (host->ops && host->ops->validate_data)
 533                 return host->ops->validate_data(host, data);
 534
 535         return 0;
 536 }
 537
 538 static int mmci_prep_data(struct mmci_host *host, struct mmc_data *data, bool next)
 539 {
 540         int err;
 541
 542         if (!host->ops || !host->ops->prep_data)
 543                 return 0;
 544
 545         err = host->ops->prep_data(host, data, next);
 546
 547         if (next && !err)
 548                 data->host_cookie = ++host->next_cookie < 0 ?
 549                         1 : host->next_cookie;
 550
 551         return err;
 552 }
 553
 554 static void mmci_unprep_data(struct mmci_host *host, struct mmc_data *data,
 555                       int err)
 556 {
 557         if (host->ops && host->ops->unprep_data)
 558                 host->ops->unprep_data(host, data, err);
 559
 560         data->host_cookie = 0;
 561 }
 562
 563 static void mmci_get_next_data(struct mmci_host *host, struct mmc_data *data)
 564 {
 565         WARN_ON(data->host_cookie && data->host_cookie != host->next_cookie);
 566
 567         if (host->ops && host->ops->get_next_data)
 568                 host->ops->get_next_data(host, data);
 569 }
 570
 571 static int mmci_dma_start(struct mmci_host *host, unsigned int datactrl)
 572 {
 573         struct mmc_data *data = host->data;
 574         int ret;
 575
 576         if (!host->use_dma)
 577                 return -EINVAL;
 578
 579         ret = mmci_prep_data(host, data, false);
 580         if (ret)
 581                 return ret;
 582
 583         if (!host->ops || !host->ops->dma_start)
 584                 return -EINVAL;
 585
 586         /* Okay, go for it. */
 587         dev_vdbg(mmc_dev(host->mmc),
 588                  "Submit MMCI DMA job, sglen %d blksz %04x blks %04x flags %08x\n",
 589                  data->sg_len, data->blksz, data->blocks, data->flags);
 590
 591         ret = host->ops->dma_start(host, &datactrl);
 592         if (ret)
 593                 return ret;
 594
 595         /* Trigger the DMA transfer */
 596         mmci_write_datactrlreg(host, datactrl);
 597
 598         /*
 599          * Let the MMCI say when the data is ended and it's time
 600          * to fire next DMA request. When that happens, MMCI will
 601          * call mmci_data_end()
 602          */
 603         writel(readl(host->base + MMCIMASK0) | MCI_DATAENDMASK,
 604                host->base + MMCIMASK0);
 605         return 0;
 606 }
 607
 608 static void mmci_dma_finalize(struct mmci_host *host, struct mmc_data *data)
 609 {
 610         if (!host->use_dma)
 611                 return;
 612
 613         if (host->ops && host->ops->dma_finalize)
 614                 host->ops->dma_finalize(host, data);
 615 }
 616
 617 static void mmci_dma_error(struct mmci_host *host)
 618 {
 619         if (!host->use_dma)
 620                 return;
 621
 622         if (host->ops && host->ops->dma_error)
 623                 host->ops->dma_error(host);
 624 }
 625
 626 static void
 627 mmci_request_end(struct mmci_host *host, struct mmc_request *mrq)
 628 {
 629         writel(0, host->base + MMCICOMMAND);
 630
 631         BUG_ON(host->data);
 632
 633         host->mrq = NULL;
 634         host->cmd = NULL;
 635
 636         mmc_request_done(host->mmc, mrq);
 637 }
 638
 639 static void mmci_set_mask1(struct mmci_host *host, unsigned int mask)
 640 {
 641         void __iomem *base = host->base;
 642         struct variant_data *variant = host->variant;
 643
 644         if (host->singleirq) {
 645                 unsigned int mask0 = readl(base + MMCIMASK0);
 646
 647                 mask0 &= ~variant->irq_pio_mask;
 648                 mask0 |= mask;
 649
 650                 writel(mask0, base + MMCIMASK0);
 651         }
 652
 653         if (variant->mmcimask1)
 654                 writel(mask, base + MMCIMASK1);
 655
 656         host->mask1_reg = mask;
 657 }
 658
 659 static void mmci_stop_data(struct mmci_host *host)
 660 {
 661         mmci_write_datactrlreg(host, 0);
 662         mmci_set_mask1(host, 0);
 663         host->data = NULL;
 664 }
 665
 666 static void mmci_init_sg(struct mmci_host *host, struct mmc_data *data)
 667 {
 668         unsigned int flags = SG_MITER_ATOMIC;
 669
 670         if (data->flags & MMC_DATA_READ)
 671                 flags |= SG_MITER_TO_SG;
 672         else
 673                 flags |= SG_MITER_FROM_SG;
 674
 675         sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 676 }
 677
 678 static u32 mmci_get_dctrl_cfg(struct mmci_host *host)
 679 {
 680         return MCI_DPSM_ENABLE | mmci_dctrl_blksz(host);
 681 }
 682
 683 static u32 ux500v2_get_dctrl_cfg(struct mmci_host *host)
 684 {
 685         return MCI_DPSM_ENABLE | (host->data->blksz << 16);
 686 }
 687
 688 static void ux500_busy_clear_mask_done(struct mmci_host *host)
 689 {
 690         void __iomem *base = host->base;
 691
 692         writel(host->variant->busy_detect_mask, base + MMCICLEAR);
 693         writel(readl(base + MMCIMASK0) &
 694                ~host->variant->busy_detect_mask, base + MMCIMASK0);
 695         host->busy_state = MMCI_BUSY_DONE;
 696         host->busy_status = 0;
 697 }
 698
 699 /*
 700  * ux500_busy_complete() - this will wait until the busy status
 701  * goes off, saving any status that occur in the meantime into
 702  * host->busy_status until we know the card is not busy any more.
 703  * The function returns true when the busy detection is ended
 704  * and we should continue processing the command.
 705  *
 706  * The Ux500 typically fires two IRQs over a busy cycle like this:
 707  *
 708  *  DAT0 busy          +-----------------+
 709  *                     |                 |
 710  *  DAT0 not busy  ----+                 +--------
 711  *
 712  *                     ^                 ^
 713  *                     |                 |
 714  *                    IRQ1              IRQ2
 715  */
 716 static bool ux500_busy_complete(struct mmci_host *host, struct mmc_command *cmd,
 717                                 u32 status, u32 err_msk)
 718 {
 719         void __iomem *base = host->base;
 720         int retries = 10;
 721
 722         if (status & err_msk) {
 723                 /* Stop any ongoing busy detection if an error occurs */
 724                 ux500_busy_clear_mask_done(host);
 725                 goto out_ret_state;
 726         }
 727
 728         /*
 729          * The state transitions are encoded in a state machine crossing
 730          * the edges in this switch statement.
 731          */
 732         switch (host->busy_state) {
 733
 734         /*
 735          * Before unmasking for the busy end IRQ, confirm that the
 736          * command was sent successfully. To keep track of having a
 737          * command in-progress, waiting for busy signaling to end,
 738          * store the status in host->busy_status.
 739          *
 740          * Note that, the card may need a couple of clock cycles before
 741          * it starts signaling busy on DAT0, hence re-read the
 742          * MMCISTATUS register here, to allow the busy bit to be set.
 743          */
 744         case MMCI_BUSY_DONE:
 745                 /*
 746                  * Save the first status register read to be sure to catch
 747                  * all bits that may be lost will retrying. If the command
 748                  * is still busy this will result in assigning 0 to
 749                  * host->busy_status, which is what it should be in IDLE.
 750                  */
 751                 host->busy_status = status & (MCI_CMDSENT | MCI_CMDRESPEND);
 752                 while (retries) {
 753                         status = readl(base + MMCISTATUS);
 754                         /* Keep accumulating status bits */
 755                         host->busy_status |= status & (MCI_CMDSENT | MCI_CMDRESPEND);
 756                         if (status & host->variant->busy_detect_flag) {
 757                                 writel(readl(base + MMCIMASK0) |
 758                                        host->variant->busy_detect_mask,
 759                                        base + MMCIMASK0);
 760                                 host->busy_state = MMCI_BUSY_WAITING_FOR_START_IRQ;
 761                                 schedule_delayed_work(&host->ux500_busy_timeout_work,
 762                                       msecs_to_jiffies(cmd->busy_timeout));
 763                                 goto out_ret_state;
 764                         }
 765                         retries--;
 766                 }
 767                 dev_dbg(mmc_dev(host->mmc), "no busy signalling in time\n");
 768                 ux500_busy_clear_mask_done(host);
 769                 break;
 770
 771         /*
 772          * If there is a command in-progress that has been successfully
 773          * sent, then bail out if busy status is set and wait for the
 774          * busy end IRQ.
 775          *
 776          * Note that, the HW triggers an IRQ on both edges while
 777          * monitoring DAT0 for busy completion, but there is only one
 778          * status bit in MMCISTATUS for the busy state. Therefore
 779          * both the start and the end interrupts needs to be cleared,
 780          * one after the other. So, clear the busy start IRQ here.
 781          */
 782         case MMCI_BUSY_WAITING_FOR_START_IRQ:
 783                 if (status & host->variant->busy_detect_flag) {
 784                         host->busy_status |= status & (MCI_CMDSENT | MCI_CMDRESPEND);
 785                         writel(host->variant->busy_detect_mask, base + MMCICLEAR);
 786                         host->busy_state = MMCI_BUSY_WAITING_FOR_END_IRQ;
 787                 } else {
 788                         dev_dbg(mmc_dev(host->mmc),
 789                                 "lost busy status when waiting for busy start IRQ\n");
 790                         cancel_delayed_work(&host->ux500_busy_timeout_work);
 791                         ux500_busy_clear_mask_done(host);
 792                 }
 793                 break;
 794
 795         case MMCI_BUSY_WAITING_FOR_END_IRQ:
 796                 if (!(status & host->variant->busy_detect_flag)) {
 797                         host->busy_status |= status & (MCI_CMDSENT | MCI_CMDRESPEND);
 798                         writel(host->variant->busy_detect_mask, base + MMCICLEAR);
 799                         cancel_delayed_work(&host->ux500_busy_timeout_work);
 800                         ux500_busy_clear_mask_done(host);
 801                 } else {
 802                         dev_dbg(mmc_dev(host->mmc),
 803                                 "busy status still asserted when handling busy end IRQ - will keep waiting\n");
 804                 }
 805                 break;
 806
 807         default:
 808                 dev_dbg(mmc_dev(host->mmc), "fell through on state %d\n",
 809                         host->busy_state);
 810                 break;
 811         }
 812
 813 out_ret_state:
 814         return (host->busy_state == MMCI_BUSY_DONE);
 815 }
 816
 817 /*
 818  * All the DMA operation mode stuff goes inside this ifdef.
 819  * This assumes that you have a generic DMA device interface,
 820  * no custom DMA interfaces are supported.
 821  */
 822 #ifdef CONFIG_DMA_ENGINE
 823 struct mmci_dmae_next {
 824         struct dma_async_tx_descriptor *desc;
 825         struct dma_chan *chan;
 826 };
 827
 828 struct mmci_dmae_priv {
 829         struct dma_chan *cur;
 830         struct dma_chan *rx_channel;
 831         struct dma_chan *tx_channel;
 832         struct dma_async_tx_descriptor  *desc_current;
 833         struct mmci_dmae_next next_data;
 834 };
 835
 836 int mmci_dmae_setup(struct mmci_host *host)
 837 {
 838         const char *rxname, *txname;
 839         struct mmci_dmae_priv *dmae;
 840
 841         dmae = devm_kzalloc(mmc_dev(host->mmc), sizeof(*dmae), GFP_KERNEL);
 842         if (!dmae)
 843                 return -ENOMEM;
 844
 845         host->dma_priv = dmae;
 846
 847         dmae->rx_channel = dma_request_chan(mmc_dev(host->mmc), "rx");
 848         if (IS_ERR(dmae->rx_channel)) {
 849                 int ret = PTR_ERR(dmae->rx_channel);
 850                 dmae->rx_channel = NULL;
 851                 return ret;
 852         }
 853
 854         dmae->tx_channel = dma_request_chan(mmc_dev(host->mmc), "tx");
 855         if (IS_ERR(dmae->tx_channel)) {
 856                 if (PTR_ERR(dmae->tx_channel) == -EPROBE_DEFER)
 857                         dev_warn(mmc_dev(host->mmc),
 858                                  "Deferred probe for TX channel ignored\n");
 859                 dmae->tx_channel = NULL;
 860         }
 861
 862         /*
 863          * If only an RX channel is specified, the driver will
 864          * attempt to use it bidirectionally, however if it
 865          * is specified but cannot be located, DMA will be disabled.
 866          */
 867         if (dmae->rx_channel && !dmae->tx_channel)
 868                 dmae->tx_channel = dmae->rx_channel;
 869
 870         if (dmae->rx_channel)
 871                 rxname = dma_chan_name(dmae->rx_channel);
 872         else
 873                 rxname = "none";
 874
 875         if (dmae->tx_channel)
 876                 txname = dma_chan_name(dmae->tx_channel);
 877         else
 878                 txname = "none";
 879
 880         dev_info(mmc_dev(host->mmc), "DMA channels RX %s, TX %s\n",
 881                  rxname, txname);
 882
 883         /*
 884          * Limit the maximum segment size in any SG entry according to
 885          * the parameters of the DMA engine device.
 886          */
 887         if (dmae->tx_channel) {
 888                 struct device *dev = dmae->tx_channel->device->dev;
 889                 unsigned int max_seg_size = dma_get_max_seg_size(dev);
 890
 891                 if (max_seg_size < host->mmc->max_seg_size)
 892                         host->mmc->max_seg_size = max_seg_size;
 893         }
 894         if (dmae->rx_channel) {
 895                 struct device *dev = dmae->rx_channel->device->dev;
 896                 unsigned int max_seg_size = dma_get_max_seg_size(dev);
 897
 898                 if (max_seg_size < host->mmc->max_seg_size)
 899                         host->mmc->max_seg_size = max_seg_size;
 900         }
 901
 902         if (!dmae->tx_channel || !dmae->rx_channel) {
 903                 mmci_dmae_release(host);
 904                 return -EINVAL;
 905         }
 906
 907         return 0;
 908 }
 909
 910 /*
 911  * This is used in or so inline it
 912  * so it can be discarded.
 913  */
 914 void mmci_dmae_release(struct mmci_host *host)
 915 {
 916         struct mmci_dmae_priv *dmae = host->dma_priv;
 917
 918         if (dmae->rx_channel)
 919                 dma_release_channel(dmae->rx_channel);
 920         if (dmae->tx_channel)
 921                 dma_release_channel(dmae->tx_channel);
 922         dmae->rx_channel = dmae->tx_channel = NULL;
 923 }
 924
 925 static void mmci_dma_unmap(struct mmci_host *host, struct mmc_data *data)
 926 {
 927         struct mmci_dmae_priv *dmae = host->dma_priv;
 928         struct dma_chan *chan;
 929
 930         if (data->flags & MMC_DATA_READ)
 931                 chan = dmae->rx_channel;
 932         else
 933                 chan = dmae->tx_channel;
 934
 935         dma_unmap_sg(chan->device->dev, data->sg, data->sg_len,
 936                      mmc_get_dma_dir(data));
 937 }
 938
 939 void mmci_dmae_error(struct mmci_host *host)
 940 {
 941         struct mmci_dmae_priv *dmae = host->dma_priv;
 942
 943         if (!dma_inprogress(host))
 944                 return;
 945
 946         dev_err(mmc_dev(host->mmc), "error during DMA transfer!\n");
 947         dmaengine_terminate_all(dmae->cur);
 948         host->dma_in_progress = false;
 949         dmae->cur = NULL;
 950         dmae->desc_current = NULL;
 951         host->data->host_cookie = 0;
 952
 953         mmci_dma_unmap(host, host->data);
 954 }
 955
 956 void mmci_dmae_finalize(struct mmci_host *host, struct mmc_data *data)
 957 {
 958         struct mmci_dmae_priv *dmae = host->dma_priv;
 959         u32 status;
 960         int i;
 961
 962         if (!dma_inprogress(host))
 963                 return;
 964
 965         /* Wait up to 1ms for the DMA to complete */
 966         for (i = 0; ; i++) {
 967                 status = readl(host->base + MMCISTATUS);
 968                 if (!(status & MCI_RXDATAAVLBLMASK) || i >= 100)
 969                         break;
 970                 udelay(10);
 971         }
 972
 973         /*
 974          * Check to see whether we still have some data left in the FIFO -
 975          * this catches DMA controllers which are unable to monitor the
 976          * DMALBREQ and DMALSREQ signals while allowing us to DMA to non-
 977          * contiguous buffers.  On TX, we'll get a FIFO underrun error.
 978          */
 979         if (status & MCI_RXDATAAVLBLMASK) {
 980                 mmci_dma_error(host);
 981                 if (!data->error)
 982                         data->error = -EIO;
 983         } else if (!data->host_cookie) {
 984                 mmci_dma_unmap(host, data);
 985         }
 986
 987         /*
 988          * Use of DMA with scatter-gather is impossible.
 989          * Give up with DMA and switch back to PIO mode.
 990          */
 991         if (status & MCI_RXDATAAVLBLMASK) {
 992                 dev_err(mmc_dev(host->mmc), "buggy DMA detected. Taking evasive action.\n");
 993                 mmci_dma_release(host);
 994         }
 995
 996         host->dma_in_progress = false;
 997         dmae->cur = NULL;
 998         dmae->desc_current = NULL;
 999 }
1000
1001 /* prepares DMA channel and DMA descriptor, returns non-zero on failure */
1002 static int _mmci_dmae_prep_data(struct mmci_host *host, struct mmc_data *data,
1003                                 struct dma_chan **dma_chan,
1004                                 struct dma_async_tx_descriptor **dma_desc)
1005 {
1006         struct mmci_dmae_priv *dmae = host->dma_priv;
1007         struct variant_data *variant = host->variant;
1008         struct dma_slave_config conf = {
1009                 .src_addr = host->phybase + MMCIFIFO,
1010                 .dst_addr = host->phybase + MMCIFIFO,
1011                 .src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
1012                 .dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
1013                 .src_maxburst = variant->fifohalfsize >> 2, /* # of words */
1014                 .dst_maxburst = variant->fifohalfsize >> 2, /* # of words */
1015                 .device_fc = false,
1016         };
1017         struct dma_chan *chan;
1018         struct dma_device *device;
1019         struct dma_async_tx_descriptor *desc;
1020         int nr_sg;
1021         unsigned long flags = DMA_CTRL_ACK;
1022
1023         if (data->flags & MMC_DATA_READ) {
1024                 conf.direction = DMA_DEV_TO_MEM;
1025                 chan = dmae->rx_channel;
1026         } else {
1027                 conf.direction = DMA_MEM_TO_DEV;
1028                 chan = dmae->tx_channel;
1029         }
1030
1031         /* If there's no DMA channel, fall back to PIO */
1032         if (!chan)
1033                 return -EINVAL;
1034
1035         /* If less than or equal to the fifo size, don't bother with DMA */
1036         if (data->blksz * data->blocks <= variant->fifosize)
1037                 return -EINVAL;
1038
1039         /*
1040          * This is necessary to get SDIO working on the Ux500. We do not yet
1041          * know if this is a bug in:
1042          * - The Ux500 DMA controller (DMA40)
1043          * - The MMCI DMA interface on the Ux500
1044          * some power of two blocks (such as 64 bytes) are sent regularly
1045          * during SDIO traffic and those work fine so for these we enable DMA
1046          * transfers.
1047          */
1048         if (host->variant->dma_power_of_2 && !is_power_of_2(data->blksz))
1049                 return -EINVAL;
1050
1051         device = chan->device;
1052         nr_sg = dma_map_sg(device->dev, data->sg, data->sg_len,
1053                            mmc_get_dma_dir(data));
1054         if (nr_sg == 0)
1055                 return -EINVAL;
1056
1057         if (host->variant->qcom_dml)
1058                 flags |= DMA_PREP_INTERRUPT;
1059
1060         dmaengine_slave_config(chan, &conf);
1061         desc = dmaengine_prep_slave_sg(chan, data->sg, nr_sg,
1062                                             conf.direction, flags);
1063         if (!desc)
1064                 goto unmap_exit;
1065
1066         *dma_chan = chan;
1067         *dma_desc = desc;
1068
1069         return 0;
1070
1071  unmap_exit:
1072         dma_unmap_sg(device->dev, data->sg, data->sg_len,
1073                      mmc_get_dma_dir(data));
1074         return -ENOMEM;
1075 }
1076
1077 int mmci_dmae_prep_data(struct mmci_host *host,
1078                         struct mmc_data *data,
1079                         bool next)
1080 {
1081         struct mmci_dmae_priv *dmae = host->dma_priv;
1082         struct mmci_dmae_next *nd = &dmae->next_data;
1083
1084         if (!host->use_dma)
1085                 return -EINVAL;
1086
1087         if (next)
1088                 return _mmci_dmae_prep_data(host, data, &nd->chan, &nd->desc);
1089         /* Check if next job is already prepared. */
1090         if (dmae->cur && dmae->desc_current)
1091                 return 0;
1092
1093         /* No job were prepared thus do it now. */
1094         return _mmci_dmae_prep_data(host, data, &dmae->cur,
1095                                     &dmae->desc_current);
1096 }
1097
1098 int mmci_dmae_start(struct mmci_host *host, unsigned int *datactrl)
1099 {
1100         struct mmci_dmae_priv *dmae = host->dma_priv;
1101         int ret;
1102
1103         host->dma_in_progress = true;
1104         ret = dma_submit_error(dmaengine_submit(dmae->desc_current));
1105         if (ret < 0) {
1106                 host->dma_in_progress = false;
1107                 return ret;
1108         }
1109         dma_async_issue_pending(dmae->cur);
1110
1111         *datactrl |= MCI_DPSM_DMAENABLE;
1112
1113         return 0;
1114 }
1115
1116 void mmci_dmae_get_next_data(struct mmci_host *host, struct mmc_data *data)
1117 {
1118         struct mmci_dmae_priv *dmae = host->dma_priv;
1119         struct mmci_dmae_next *next = &dmae->next_data;
1120
1121         if (!host->use_dma)
1122                 return;
1123
1124         WARN_ON(!data->host_cookie && (next->desc || next->chan));
1125
1126         dmae->desc_current = next->desc;
1127         dmae->cur = next->chan;
1128         next->desc = NULL;
1129         next->chan = NULL;
1130 }
1131
1132 void mmci_dmae_unprep_data(struct mmci_host *host,
1133                            struct mmc_data *data, int err)
1134
1135 {
1136         struct mmci_dmae_priv *dmae = host->dma_priv;
1137
1138         if (!host->use_dma)
1139                 return;
1140
1141         mmci_dma_unmap(host, data);
1142
1143         if (err) {
1144                 struct mmci_dmae_next *next = &dmae->next_data;
1145                 struct dma_chan *chan;
1146                 if (data->flags & MMC_DATA_READ)
1147                         chan = dmae->rx_channel;
1148                 else
1149                         chan = dmae->tx_channel;
1150                 dmaengine_terminate_all(chan);
1151
1152                 if (dmae->desc_current == next->desc)
1153                         dmae->desc_current = NULL;
1154
1155                 if (dmae->cur == next->chan) {
1156                         host->dma_in_progress = false;
1157                         dmae->cur = NULL;
1158                 }
1159
1160                 next->desc = NULL;
1161                 next->chan = NULL;
1162         }
1163 }
1164
1165 static struct mmci_host_ops mmci_variant_ops = {
1166         .prep_data = mmci_dmae_prep_data,
1167         .unprep_data = mmci_dmae_unprep_data,
1168         .get_datactrl_cfg = mmci_get_dctrl_cfg,
1169         .get_next_data = mmci_dmae_get_next_data,
1170         .dma_setup = mmci_dmae_setup,
1171         .dma_release = mmci_dmae_release,
1172         .dma_start = mmci_dmae_start,
1173         .dma_finalize = mmci_dmae_finalize,
1174         .dma_error = mmci_dmae_error,
1175 };
1176 #else
1177 static struct mmci_host_ops mmci_variant_ops = {
1178         .get_datactrl_cfg = mmci_get_dctrl_cfg,
1179 };
1180 #endif
1181
1182 static void mmci_variant_init(struct mmci_host *host)
1183 {
1184         host->ops = &mmci_variant_ops;
1185 }
1186
1187 static void ux500_variant_init(struct mmci_host *host)
1188 {
1189         host->ops = &mmci_variant_ops;
1190         host->ops->busy_complete = ux500_busy_complete;
1191 }
1192
1193 static void ux500v2_variant_init(struct mmci_host *host)
1194 {
1195         host->ops = &mmci_variant_ops;
1196         host->ops->busy_complete = ux500_busy_complete;
1197         host->ops->get_datactrl_cfg = ux500v2_get_dctrl_cfg;
1198 }
1199
1200 static void mmci_pre_request(struct mmc_host *mmc, struct mmc_request *mrq)
1201 {
1202         struct mmci_host *host = mmc_priv(mmc);
1203         struct mmc_data *data = mrq->data;
1204
1205         if (!data)
1206                 return;
1207
1208         WARN_ON(data->host_cookie);
1209
1210         if (mmci_validate_data(host, data))
1211                 return;
1212
1213         mmci_prep_data(host, data, true);
1214 }
1215
1216 static void mmci_post_request(struct mmc_host *mmc, struct mmc_request *mrq,
1217                               int err)
1218 {
1219         struct mmci_host *host = mmc_priv(mmc);
1220         struct mmc_data *data = mrq->data;
1221
1222         if (!data || !data->host_cookie)
1223                 return;
1224
1225         mmci_unprep_data(host, data, err);
1226 }
1227
1228 static void mmci_start_data(struct mmci_host *host, struct mmc_data *data)
1229 {
1230         struct variant_data *variant = host->variant;
1231         unsigned int datactrl, timeout, irqmask;
1232         unsigned long long clks;
1233         void __iomem *base;
1234
1235         dev_dbg(mmc_dev(host->mmc), "blksz %04x blks %04x flags %08x\n",
1236                 data->blksz, data->blocks, data->flags);
1237
1238         host->data = data;
1239         host->size = data->blksz * data->blocks;
1240         data->bytes_xfered = 0;
1241
1242         clks = (unsigned long long)data->timeout_ns * host->cclk;
1243         do_div(clks, NSEC_PER_SEC);
1244
1245         timeout = data->timeout_clks + (unsigned int)clks;
1246
1247         base = host->base;
1248         writel(timeout, base + MMCIDATATIMER);
1249         writel(host->size, base + MMCIDATALENGTH);
1250
1251         datactrl = host->ops->get_datactrl_cfg(host);
1252         datactrl |= host->data->flags & MMC_DATA_READ ? MCI_DPSM_DIRECTION : 0;
1253
1254         if (host->mmc->card && mmc_card_sdio(host->mmc->card)) {
1255                 u32 clk;
1256
1257                 datactrl |= variant->datactrl_mask_sdio;
1258
1259                 /*
1260                  * The ST Micro variant for SDIO small write transfers
1261                  * needs to have clock H/W flow control disabled,
1262                  * otherwise the transfer will not start. The threshold
1263                  * depends on the rate of MCLK.
1264                  */
1265                 if (variant->st_sdio && data->flags & MMC_DATA_WRITE &&
1266                     (host->size < 8 ||
1267                      (host->size <= 8 && host->mclk > 50000000)))
1268                         clk = host->clk_reg & ~variant->clkreg_enable;
1269                 else
1270                         clk = host->clk_reg | variant->clkreg_enable;
1271
1272                 mmci_write_clkreg(host, clk);
1273         }
1274
1275         if (host->mmc->ios.timing == MMC_TIMING_UHS_DDR50 ||
1276             host->mmc->ios.timing == MMC_TIMING_MMC_DDR52)
1277                 datactrl |= variant->datactrl_mask_ddrmode;
1278
1279         /*
1280          * Attempt to use DMA operation mode, if this
1281          * should fail, fall back to PIO mode
1282          */
1283         if (!mmci_dma_start(host, datactrl))
1284                 return;
1285
1286         /* IRQ mode, map the SG list for CPU reading/writing */
1287         mmci_init_sg(host, data);
1288
1289         if (data->flags & MMC_DATA_READ) {
1290                 irqmask = MCI_RXFIFOHALFFULLMASK;
1291
1292                 /*
1293                  * If we have less than the fifo 'half-full' threshold to
1294                  * transfer, trigger a PIO interrupt as soon as any data
1295                  * is available.
1296                  */
1297                 if (host->size < variant->fifohalfsize)
1298                         irqmask |= MCI_RXDATAAVLBLMASK;
1299         } else {
1300                 /*
1301                  * We don't actually need to include "FIFO empty" here
1302                  * since its implicit in "FIFO half empty".
1303                  */
1304                 irqmask = MCI_TXFIFOHALFEMPTYMASK;
1305         }
1306
1307         mmci_write_datactrlreg(host, datactrl);
1308         writel(readl(base + MMCIMASK0) & ~MCI_DATAENDMASK, base + MMCIMASK0);
1309         mmci_set_mask1(host, irqmask);
1310 }
1311
1312 static void
1313 mmci_start_command(struct mmci_host *host, struct mmc_command *cmd, u32 c)
1314 {
1315         void __iomem *base = host->base;
1316         bool busy_resp = cmd->flags & MMC_RSP_BUSY;
1317         unsigned long long clks;
1318
1319         dev_dbg(mmc_dev(host->mmc), "op %02x arg %08x flags %08x\n",
1320             cmd->opcode, cmd->arg, cmd->flags);
1321
1322         if (readl(base + MMCICOMMAND) & host->variant->cmdreg_cpsm_enable) {
1323                 writel(0, base + MMCICOMMAND);
1324                 mmci_reg_delay(host);
1325         }
1326
1327         if (host->variant->cmdreg_stop &&
1328             cmd->opcode == MMC_STOP_TRANSMISSION)
1329                 c |= host->variant->cmdreg_stop;
1330
1331         c |= cmd->opcode | host->variant->cmdreg_cpsm_enable;
1332         if (cmd->flags & MMC_RSP_PRESENT) {
1333                 if (cmd->flags & MMC_RSP_136)
1334                         c |= host->variant->cmdreg_lrsp_crc;
1335                 else if (cmd->flags & MMC_RSP_CRC)
1336                         c |= host->variant->cmdreg_srsp_crc;
1337                 else
1338                         c |= host->variant->cmdreg_srsp;
1339         }
1340
1341         host->busy_status = 0;
1342         host->busy_state = MMCI_BUSY_DONE;
1343
1344         /* Assign a default timeout if the core does not provide one */
1345         if (busy_resp && !cmd->busy_timeout)
1346                 cmd->busy_timeout = 10 * MSEC_PER_SEC;
1347
1348         if (busy_resp && host->variant->busy_timeout) {
1349                 if (cmd->busy_timeout > host->mmc->max_busy_timeout)
1350                         clks = (unsigned long long)host->mmc->max_busy_timeout * host->cclk;
1351                 else
1352                         clks = (unsigned long long)cmd->busy_timeout * host->cclk;
1353
1354                 do_div(clks, MSEC_PER_SEC);
1355                 writel_relaxed(clks, host->base + MMCIDATATIMER);
1356         }
1357
1358         if (host->ops->pre_sig_volt_switch && cmd->opcode == SD_SWITCH_VOLTAGE)
1359                 host->ops->pre_sig_volt_switch(host);
1360
1361         if (/*interrupt*/0)
1362                 c |= MCI_CPSM_INTERRUPT;
1363
1364         if (mmc_cmd_type(cmd) == MMC_CMD_ADTC)
1365                 c |= host->variant->data_cmd_enable;
1366
1367         host->cmd = cmd;
1368
1369         writel(cmd->arg, base + MMCIARGUMENT);
1370         writel(c, base + MMCICOMMAND);
1371 }
1372
1373 static void mmci_stop_command(struct mmci_host *host)
1374 {
1375         host->stop_abort.error = 0;
1376         mmci_start_command(host, &host->stop_abort, 0);
1377 }
1378
1379 static void
1380 mmci_data_irq(struct mmci_host *host, struct mmc_data *data,
1381               unsigned int status)
1382 {
1383         unsigned int status_err;
1384
1385         /* Make sure we have data to handle */
1386         if (!data)
1387                 return;
1388
1389         /* First check for errors */
1390         status_err = status & (host->variant->start_err |
1391                                MCI_DATACRCFAIL | MCI_DATATIMEOUT |
1392                                MCI_TXUNDERRUN | MCI_RXOVERRUN);
1393
1394         if (status_err) {
1395                 u32 remain, success;
1396
1397                 /* Terminate the DMA transfer */
1398                 mmci_dma_error(host);
1399
1400                 /*
1401                  * Calculate how far we are into the transfer.  Note that
1402                  * the data counter gives the number of bytes transferred
1403                  * on the MMC bus, not on the host side.  On reads, this
1404                  * can be as much as a FIFO-worth of data ahead.  This
1405                  * matters for FIFO overruns only.
1406                  */
1407                 if (!host->variant->datacnt_useless) {
1408                         remain = readl(host->base + MMCIDATACNT);
1409                         success = data->blksz * data->blocks - remain;
1410                 } else {
1411                         success = 0;
1412                 }
1413
1414                 dev_dbg(mmc_dev(host->mmc), "MCI ERROR IRQ, status 0x%08x at 0x%08x\n",
1415                         status_err, success);
1416                 if (status_err & MCI_DATACRCFAIL) {
1417                         /* Last block was not successful */
1418                         success -= 1;
1419                         data->error = -EILSEQ;
1420                 } else if (status_err & MCI_DATATIMEOUT) {
1421                         data->error = -ETIMEDOUT;
1422                 } else if (status_err & MCI_STARTBITERR) {
1423                         data->error = -ECOMM;
1424                 } else if (status_err & MCI_TXUNDERRUN) {
1425                         data->error = -EIO;
1426                 } else if (status_err & MCI_RXOVERRUN) {
1427                         if (success > host->variant->fifosize)
1428                                 success -= host->variant->fifosize;
1429                         else
1430                                 success = 0;
1431                         data->error = -EIO;
1432                 }
1433                 data->bytes_xfered = round_down(success, data->blksz);
1434         }
1435
1436         if (status & MCI_DATABLOCKEND)
1437                 dev_err(mmc_dev(host->mmc), "stray MCI_DATABLOCKEND interrupt\n");
1438
1439         if (status & MCI_DATAEND || data->error) {
1440                 mmci_dma_finalize(host, data);
1441
1442                 mmci_stop_data(host);
1443
1444                 if (!data->error)
1445                         /* The error clause is handled above, success! */
1446                         data->bytes_xfered = data->blksz * data->blocks;
1447
1448                 if (!data->stop) {
1449                         if (host->variant->cmdreg_stop && data->error)
1450                                 mmci_stop_command(host);
1451                         else
1452                                 mmci_request_end(host, data->mrq);
1453                 } else if (host->mrq->sbc && !data->error) {
1454                         mmci_request_end(host, data->mrq);
1455                 } else {
1456                         mmci_start_command(host, data->stop, 0);
1457                 }
1458         }
1459 }
1460
1461 static void
1462 mmci_cmd_irq(struct mmci_host *host, struct mmc_command *cmd,
1463              unsigned int status)
1464 {
1465         u32 err_msk = MCI_CMDCRCFAIL | MCI_CMDTIMEOUT;
1466         void __iomem *base = host->base;
1467         bool sbc, busy_resp;
1468
1469         if (!cmd)
1470                 return;
1471
1472         sbc = (cmd == host->mrq->sbc);
1473         busy_resp = !!(cmd->flags & MMC_RSP_BUSY);
1474
1475         /*
1476          * We need to be one of these interrupts to be considered worth
1477          * handling. Note that we tag on any latent IRQs postponed
1478          * due to waiting for busy status.
1479          */
1480         if (host->variant->busy_timeout && busy_resp)
1481                 err_msk |= MCI_DATATIMEOUT;
1482
1483         if (!((status | host->busy_status) &
1484               (err_msk | MCI_CMDSENT | MCI_CMDRESPEND)))
1485                 return;
1486
1487         /* Handle busy detection on DAT0 if the variant supports it. */
1488         if (busy_resp && host->variant->busy_detect)
1489                 if (!host->ops->busy_complete(host, cmd, status, err_msk))
1490                         return;
1491
1492         host->cmd = NULL;
1493
1494         if (status & MCI_CMDTIMEOUT) {
1495                 cmd->error = -ETIMEDOUT;
1496         } else if (status & MCI_CMDCRCFAIL && cmd->flags & MMC_RSP_CRC) {
1497                 cmd->error = -EILSEQ;
1498         } else if (host->variant->busy_timeout && busy_resp &&
1499                    status & MCI_DATATIMEOUT) {
1500                 cmd->error = -ETIMEDOUT;
1501                 /*
1502                  * This will wake up mmci_irq_thread() which will issue
1503                  * a hardware reset of the MMCI block.
1504                  */
1505                 host->irq_action = IRQ_WAKE_THREAD;
1506         } else {
1507                 cmd->resp[0] = readl(base + MMCIRESPONSE0);
1508                 cmd->resp[1] = readl(base + MMCIRESPONSE1);
1509                 cmd->resp[2] = readl(base + MMCIRESPONSE2);
1510                 cmd->resp[3] = readl(base + MMCIRESPONSE3);
1511         }
1512
1513         if ((!sbc && !cmd->data) || cmd->error) {
1514                 if (host->data) {
1515                         /* Terminate the DMA transfer */
1516                         mmci_dma_error(host);
1517
1518                         mmci_stop_data(host);
1519                         if (host->variant->cmdreg_stop && cmd->error) {
1520                                 mmci_stop_command(host);
1521                                 return;
1522                         }
1523                 }
1524
1525                 if (host->irq_action != IRQ_WAKE_THREAD)
1526                         mmci_request_end(host, host->mrq);
1527
1528         } else if (sbc) {
1529                 mmci_start_command(host, host->mrq->cmd, 0);
1530         } else if (!host->variant->datactrl_first &&
1531                    !(cmd->data->flags & MMC_DATA_READ)) {
1532                 mmci_start_data(host, cmd->data);
1533         }
1534 }
1535
1536 /*
1537  * This busy timeout worker is used to "kick" the command IRQ if a
1538  * busy detect IRQ fails to appear in reasonable time. Only used on
1539  * variants with busy detection IRQ delivery.
1540  */
1541 static void ux500_busy_timeout_work(struct work_struct *work)
1542 {
1543         struct mmci_host *host = container_of(work, struct mmci_host,
1544                                         ux500_busy_timeout_work.work);
1545         unsigned long flags;
1546         u32 status;
1547
1548         spin_lock_irqsave(&host->lock, flags);
1549
1550         if (host->cmd) {
1551                 dev_dbg(mmc_dev(host->mmc), "timeout waiting for busy IRQ\n");
1552
1553                 /* If we are still busy let's tag on a cmd-timeout error. */
1554                 status = readl(host->base + MMCISTATUS);
1555                 if (status & host->variant->busy_detect_flag)
1556                         status |= MCI_CMDTIMEOUT;
1557
1558                 mmci_cmd_irq(host, host->cmd, status);
1559         }
1560
1561         spin_unlock_irqrestore(&host->lock, flags);
1562 }
1563
1564 static int mmci_get_rx_fifocnt(struct mmci_host *host, u32 status, int remain)
1565 {
1566         return remain - (readl(host->base + MMCIFIFOCNT) << 2);
1567 }
1568
1569 static int mmci_qcom_get_rx_fifocnt(struct mmci_host *host, u32 status, int r)
1570 {
1571         /*
1572          * on qcom SDCC4 only 8 words are used in each burst so only 8 addresses
1573          * from the fifo range should be used
1574          */
1575         if (status & MCI_RXFIFOHALFFULL)
1576                 return host->variant->fifohalfsize;
1577         else if (status & MCI_RXDATAAVLBL)
1578                 return 4;
1579
1580         return 0;
1581 }
1582
1583 static int mmci_pio_read(struct mmci_host *host, char *buffer, unsigned int remain)
1584 {
1585         void __iomem *base = host->base;
1586         char *ptr = buffer;
1587         u32 status = readl(host->base + MMCISTATUS);
1588         int host_remain = host->size;
1589
1590         do {
1591                 int count = host->get_rx_fifocnt(host, status, host_remain);
1592
1593                 if (count > remain)
1594                         count = remain;
1595
1596                 if (count <= 0)
1597                         break;
1598
1599                 /*
1600                  * SDIO especially may want to send something that is
1601                  * not divisible by 4 (as opposed to card sectors
1602                  * etc). Therefore make sure to always read the last bytes
1603                  * while only doing full 32-bit reads towards the FIFO.
1604                  */
1605                 if (unlikely(count & 0x3)) {
1606                         if (count < 4) {
1607                                 unsigned char buf[4];
1608                                 ioread32_rep(base + MMCIFIFO, buf, 1);
1609                                 memcpy(ptr, buf, count);
1610                         } else {
1611                                 ioread32_rep(base + MMCIFIFO, ptr, count >> 2);
1612                                 count &= ~0x3;
1613                         }
1614                 } else {
1615                         ioread32_rep(base + MMCIFIFO, ptr, count >> 2);
1616                 }
1617
1618                 ptr += count;
1619                 remain -= count;
1620                 host_remain -= count;
1621
1622                 if (remain == 0)
1623                         break;
1624
1625                 status = readl(base + MMCISTATUS);
1626         } while (status & MCI_RXDATAAVLBL);
1627
1628         return ptr - buffer;
1629 }
1630
1631 static int mmci_pio_write(struct mmci_host *host, char *buffer, unsigned int remain, u32 status)
1632 {
1633         struct variant_data *variant = host->variant;
1634         void __iomem *base = host->base;
1635         char *ptr = buffer;
1636
1637         do {
1638                 unsigned int count, maxcnt;
1639
1640                 maxcnt = status & MCI_TXFIFOEMPTY ?
1641                          variant->fifosize : variant->fifohalfsize;
1642                 count = min(remain, maxcnt);
1643
1644                 /*
1645                  * SDIO especially may want to send something that is
1646                  * not divisible by 4 (as opposed to card sectors
1647                  * etc), and the FIFO only accept full 32-bit writes.
1648                  * So compensate by adding +3 on the count, a single
1649                  * byte become a 32bit write, 7 bytes will be two
1650                  * 32bit writes etc.
1651                  */
1652                 iowrite32_rep(base + MMCIFIFO, ptr, (count + 3) >> 2);
1653
1654                 ptr += count;
1655                 remain -= count;
1656
1657                 if (remain == 0)
1658                         break;
1659
1660                 status = readl(base + MMCISTATUS);
1661         } while (status & MCI_TXFIFOHALFEMPTY);
1662
1663         return ptr - buffer;
1664 }
1665
1666 /*
1667  * PIO data transfer IRQ handler.
1668  */
1669 static irqreturn_t mmci_pio_irq(int irq, void *dev_id)
1670 {
1671         struct mmci_host *host = dev_id;
1672         struct sg_mapping_iter *sg_miter = &host->sg_miter;
1673         struct variant_data *variant = host->variant;
1674         void __iomem *base = host->base;
1675         u32 status;
1676
1677         status = readl(base + MMCISTATUS);
1678
1679         dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status);
1680
1681         do {
1682                 unsigned int remain, len;
1683                 char *buffer;
1684
1685                 /*
1686                  * For write, we only need to test the half-empty flag
1687                  * here - if the FIFO is completely empty, then by
1688                  * definition it is more than half empty.
1689                  *
1690                  * For read, check for data available.
1691                  */
1692                 if (!(status & (MCI_TXFIFOHALFEMPTY|MCI_RXDATAAVLBL)))
1693                         break;
1694
1695                 if (!sg_miter_next(sg_miter))
1696                         break;
1697
1698                 buffer = sg_miter->addr;
1699                 remain = sg_miter->length;
1700
1701                 len = 0;
1702                 if (status & MCI_RXACTIVE)
1703                         len = mmci_pio_read(host, buffer, remain);
1704                 if (status & MCI_TXACTIVE)
1705                         len = mmci_pio_write(host, buffer, remain, status);
1706
1707                 sg_miter->consumed = len;
1708
1709                 host->size -= len;
1710                 remain -= len;
1711
1712                 if (remain)
1713                         break;
1714
1715                 status = readl(base + MMCISTATUS);
1716         } while (1);
1717
1718         sg_miter_stop(sg_miter);
1719
1720         /*
1721          * If we have less than the fifo 'half-full' threshold to transfer,
1722          * trigger a PIO interrupt as soon as any data is available.
1723          */
1724         if (status & MCI_RXACTIVE && host->size < variant->fifohalfsize)
1725                 mmci_set_mask1(host, MCI_RXDATAAVLBLMASK);
1726
1727         /*
1728          * If we run out of data, disable the data IRQs; this
1729          * prevents a race where the FIFO becomes empty before
1730          * the chip itself has disabled the data path, and
1731          * stops us racing with our data end IRQ.
1732          */
1733         if (host->size == 0) {
1734                 mmci_set_mask1(host, 0);
1735                 writel(readl(base + MMCIMASK0) | MCI_DATAENDMASK, base + MMCIMASK0);
1736         }
1737
1738         return IRQ_HANDLED;
1739 }
1740
1741 /*
1742  * Handle completion of command and data transfers.
1743  */
1744 static irqreturn_t mmci_irq(int irq, void *dev_id)
1745 {
1746         struct mmci_host *host = dev_id;
1747         u32 status;
1748
1749         spin_lock(&host->lock);
1750         host->irq_action = IRQ_HANDLED;
1751
1752         do {
1753                 status = readl(host->base + MMCISTATUS);
1754                 if (!status)
1755                         break;
1756
1757                 if (host->singleirq) {
1758                         if (status & host->mask1_reg)
1759                                 mmci_pio_irq(irq, dev_id);
1760
1761                         status &= ~host->variant->irq_pio_mask;
1762                 }
1763
1764                 /*
1765                  * Busy detection is managed by mmci_cmd_irq(), including to
1766                  * clear the corresponding IRQ.
1767                  */
1768                 status &= readl(host->base + MMCIMASK0);
1769                 if (host->variant->busy_detect)
1770                         writel(status & ~host->variant->busy_detect_mask,
1771                                host->base + MMCICLEAR);
1772                 else
1773                         writel(status, host->base + MMCICLEAR);
1774
1775                 dev_dbg(mmc_dev(host->mmc), "irq0 (data+cmd) %08x\n", status);
1776
1777                 if (host->variant->reversed_irq_handling) {
1778                         mmci_data_irq(host, host->data, status);
1779                         mmci_cmd_irq(host, host->cmd, status);
1780                 } else {
1781                         mmci_cmd_irq(host, host->cmd, status);
1782                         mmci_data_irq(host, host->data, status);
1783                 }
1784
1785                 /*
1786                  * Busy detection has been handled by mmci_cmd_irq() above.
1787                  * Clear the status bit to prevent polling in IRQ context.
1788                  */
1789                 if (host->variant->busy_detect_flag)
1790                         status &= ~host->variant->busy_detect_flag;
1791
1792         } while (status);
1793
1794         spin_unlock(&host->lock);
1795
1796         return host->irq_action;
1797 }
1798
1799 /*
1800  * mmci_irq_thread() - A threaded IRQ handler that manages a reset of the HW.
1801  *
1802  * A reset is needed for some variants, where a datatimeout for a R1B request
1803  * causes the DPSM to stay busy (non-functional).
1804  */
1805 static irqreturn_t mmci_irq_thread(int irq, void *dev_id)
1806 {
1807         struct mmci_host *host = dev_id;
1808         unsigned long flags;
1809
1810         if (host->rst) {
1811                 reset_control_assert(host->rst);
1812                 udelay(2);
1813                 reset_control_deassert(host->rst);
1814         }
1815
1816         spin_lock_irqsave(&host->lock, flags);
1817         writel(host->clk_reg, host->base + MMCICLOCK);
1818         writel(host->pwr_reg, host->base + MMCIPOWER);
1819         writel(MCI_IRQENABLE | host->variant->start_err,
1820                host->base + MMCIMASK0);
1821
1822         host->irq_action = IRQ_HANDLED;
1823         mmci_request_end(host, host->mrq);
1824         spin_unlock_irqrestore(&host->lock, flags);
1825
1826         return host->irq_action;
1827 }
1828
1829 static void mmci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1830 {
1831         struct mmci_host *host = mmc_priv(mmc);
1832         unsigned long flags;
1833
1834         WARN_ON(host->mrq != NULL);
1835
1836         mrq->cmd->error = mmci_validate_data(host, mrq->data);
1837         if (mrq->cmd->error) {
1838                 mmc_request_done(mmc, mrq);
1839                 return;
1840         }
1841
1842         spin_lock_irqsave(&host->lock, flags);
1843
1844         host->mrq = mrq;
1845
1846         if (mrq->data)
1847                 mmci_get_next_data(host, mrq->data);
1848
1849         if (mrq->data &&
1850             (host->variant->datactrl_first || mrq->data->flags & MMC_DATA_READ))
1851                 mmci_start_data(host, mrq->data);
1852
1853         if (mrq->sbc)
1854                 mmci_start_command(host, mrq->sbc, 0);
1855         else
1856                 mmci_start_command(host, mrq->cmd, 0);
1857
1858         spin_unlock_irqrestore(&host->lock, flags);
1859 }
1860
1861 static void mmci_set_max_busy_timeout(struct mmc_host *mmc)
1862 {
1863         struct mmci_host *host = mmc_priv(mmc);
1864         u32 max_busy_timeout = 0;
1865
1866         if (!host->variant->busy_detect)
1867                 return;
1868
1869         if (host->variant->busy_timeout && mmc->actual_clock)
1870                 max_busy_timeout = U32_MAX / DIV_ROUND_UP(mmc->actual_clock,
1871                                                           MSEC_PER_SEC);
1872
1873         mmc->max_busy_timeout = max_busy_timeout;
1874 }
1875
1876 static void mmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1877 {
1878         struct mmci_host *host = mmc_priv(mmc);
1879         struct variant_data *variant = host->variant;
1880         u32 pwr = 0;
1881         unsigned long flags;
1882         int ret;
1883
1884         switch (ios->power_mode) {
1885         case MMC_POWER_OFF:
1886                 if (!IS_ERR(mmc->supply.vmmc))
1887                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1888
1889                 if (!IS_ERR(mmc->supply.vqmmc) && host->vqmmc_enabled) {
1890                         regulator_disable(mmc->supply.vqmmc);
1891                         host->vqmmc_enabled = false;
1892                 }
1893
1894                 break;
1895         case MMC_POWER_UP:
1896                 if (!IS_ERR(mmc->supply.vmmc))
1897                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
1898
1899                 /*
1900                  * The ST Micro variant doesn't have the PL180s MCI_PWR_UP
1901                  * and instead uses MCI_PWR_ON so apply whatever value is
1902                  * configured in the variant data.
1903                  */
1904                 pwr |= variant->pwrreg_powerup;
1905
1906                 break;
1907         case MMC_POWER_ON:
1908                 if (!IS_ERR(mmc->supply.vqmmc) && !host->vqmmc_enabled) {
1909                         ret = regulator_enable(mmc->supply.vqmmc);
1910                         if (ret < 0)
1911                                 dev_err(mmc_dev(mmc),
1912                                         "failed to enable vqmmc regulator\n");
1913                         else
1914                                 host->vqmmc_enabled = true;
1915                 }
1916
1917                 pwr |= MCI_PWR_ON;
1918                 break;
1919         }
1920
1921         if (variant->signal_direction && ios->power_mode != MMC_POWER_OFF) {
1922                 /*
1923                  * The ST Micro variant has some additional bits
1924                  * indicating signal direction for the signals in
1925                  * the SD/MMC bus and feedback-clock usage.
1926                  */
1927                 pwr |= host->pwr_reg_add;
1928
1929                 if (ios->bus_width == MMC_BUS_WIDTH_4)
1930                         pwr &= ~MCI_ST_DATA74DIREN;
1931                 else if (ios->bus_width == MMC_BUS_WIDTH_1)
1932                         pwr &= (~MCI_ST_DATA74DIREN &
1933                                 ~MCI_ST_DATA31DIREN &
1934                                 ~MCI_ST_DATA2DIREN);
1935         }
1936
1937         if (variant->opendrain) {
1938                 if (ios->bus_mode == MMC_BUSMODE_OPENDRAIN)
1939                         pwr |= variant->opendrain;
1940         } else {
1941                 /*
1942                  * If the variant cannot configure the pads by its own, then we
1943                  * expect the pinctrl to be able to do that for us
1944                  */
1945                 if (ios->bus_mode == MMC_BUSMODE_OPENDRAIN)
1946                         pinctrl_select_state(host->pinctrl, host->pins_opendrain);
1947                 else
1948                         pinctrl_select_default_state(mmc_dev(mmc));
1949         }
1950
1951         /*
1952          * If clock = 0 and the variant requires the MMCIPOWER to be used for
1953          * gating the clock, the MCI_PWR_ON bit is cleared.
1954          */
1955         if (!ios->clock && variant->pwrreg_clkgate)
1956                 pwr &= ~MCI_PWR_ON;
1957
1958         if (host->variant->explicit_mclk_control &&
1959             ios->clock != host->clock_cache) {
1960                 ret = clk_set_rate(host->clk, ios->clock);
1961                 if (ret < 0)
1962                         dev_err(mmc_dev(host->mmc),
1963                                 "Error setting clock rate (%d)\n", ret);
1964                 else
1965                         host->mclk = clk_get_rate(host->clk);
1966         }
1967         host->clock_cache = ios->clock;
1968
1969         spin_lock_irqsave(&host->lock, flags);
1970
1971         if (host->ops && host->ops->set_clkreg)
1972                 host->ops->set_clkreg(host, ios->clock);
1973         else
1974                 mmci_set_clkreg(host, ios->clock);
1975
1976         mmci_set_max_busy_timeout(mmc);
1977
1978         if (host->ops && host->ops->set_pwrreg)
1979                 host->ops->set_pwrreg(host, pwr);
1980         else
1981                 mmci_write_pwrreg(host, pwr);
1982
1983         mmci_reg_delay(host);
1984
1985         spin_unlock_irqrestore(&host->lock, flags);
1986 }
1987
1988 static int mmci_get_cd(struct mmc_host *mmc)
1989 {
1990         struct mmci_host *host = mmc_priv(mmc);
1991         struct mmci_platform_data *plat = host->plat;
1992         unsigned int status = mmc_gpio_get_cd(mmc);
1993
1994         if (status == -ENOSYS) {
1995                 if (!plat->status)
1996                         return 1; /* Assume always present */
1997
1998                 status = plat->status(mmc_dev(host->mmc));
1999         }
2000         return status;
2001 }
2002
2003 static int mmci_sig_volt_switch(struct mmc_host *mmc, struct mmc_ios *ios)
2004 {
2005         struct mmci_host *host = mmc_priv(mmc);
2006         int ret;
2007
2008         ret = mmc_regulator_set_vqmmc(mmc, ios);
2009
2010         if (!ret && host->ops && host->ops->post_sig_volt_switch)
2011                 ret = host->ops->post_sig_volt_switch(host, ios);
2012         else if (ret)
2013                 ret = 0;
2014
2015         if (ret < 0)
2016                 dev_warn(mmc_dev(mmc), "Voltage switch failed\n");
2017
2018         return ret;
2019 }
2020
2021 static struct mmc_host_ops mmci_ops = {
2022         .request        = mmci_request,
2023         .pre_req        = mmci_pre_request,
2024         .post_req       = mmci_post_request,
2025         .set_ios        = mmci_set_ios,
2026         .get_ro         = mmc_gpio_get_ro,
2027         .get_cd         = mmci_get_cd,
2028         .start_signal_voltage_switch = mmci_sig_volt_switch,
2029 };
2030
2031 static void mmci_probe_level_translator(struct mmc_host *mmc)
2032 {
2033         struct device *dev = mmc_dev(mmc);
2034         struct mmci_host *host = mmc_priv(mmc);
2035         struct gpio_desc *cmd_gpio;
2036         struct gpio_desc *ck_gpio;
2037         struct gpio_desc *ckin_gpio;
2038         int clk_hi, clk_lo;
2039
2040         /*
2041          * Assume the level translator is present if st,use-ckin is set.
2042          * This is to cater for DTs which do not implement this test.
2043          */
2044         host->clk_reg_add |= MCI_STM32_CLK_SELCKIN;
2045
2046         cmd_gpio = gpiod_get(dev, "st,cmd", GPIOD_OUT_HIGH);
2047         if (IS_ERR(cmd_gpio))
2048                 goto exit_cmd;
2049
2050         ck_gpio = gpiod_get(dev, "st,ck", GPIOD_OUT_HIGH);
2051         if (IS_ERR(ck_gpio))
2052                 goto exit_ck;
2053
2054         ckin_gpio = gpiod_get(dev, "st,ckin", GPIOD_IN);
2055         if (IS_ERR(ckin_gpio))
2056                 goto exit_ckin;
2057
2058         /* All GPIOs are valid, test whether level translator works */
2059
2060         /* Sample CKIN */
2061         clk_hi = !!gpiod_get_value(ckin_gpio);
2062
2063         /* Set CK low */
2064         gpiod_set_value(ck_gpio, 0);
2065
2066         /* Sample CKIN */
2067         clk_lo = !!gpiod_get_value(ckin_gpio);
2068
2069         /* Tristate all */
2070         gpiod_direction_input(cmd_gpio);
2071         gpiod_direction_input(ck_gpio);
2072
2073         /* Level translator is present if CK signal is propagated to CKIN */
2074         if (!clk_hi || clk_lo) {
2075                 host->clk_reg_add &= ~MCI_STM32_CLK_SELCKIN;
2076                 dev_warn(dev,
2077                          "Level translator inoperable, CK signal not detected on CKIN, disabling.\n");
2078         }
2079
2080         gpiod_put(ckin_gpio);
2081
2082 exit_ckin:
2083         gpiod_put(ck_gpio);
2084 exit_ck:
2085         gpiod_put(cmd_gpio);
2086 exit_cmd:
2087         pinctrl_select_default_state(dev);
2088 }
2089
2090 static int mmci_of_parse(struct device_node *np, struct mmc_host *mmc)
2091 {
2092         struct mmci_host *host = mmc_priv(mmc);
2093         int ret = mmc_of_parse(mmc);
2094
2095         if (ret)
2096                 return ret;
2097
2098         if (of_property_read_bool(np, "st,sig-dir-dat0"))
2099                 host->pwr_reg_add |= MCI_ST_DATA0DIREN;
2100         if (of_property_read_bool(np, "st,sig-dir-dat2"))
2101                 host->pwr_reg_add |= MCI_ST_DATA2DIREN;
2102         if (of_property_read_bool(np, "st,sig-dir-dat31"))
2103                 host->pwr_reg_add |= MCI_ST_DATA31DIREN;
2104         if (of_property_read_bool(np, "st,sig-dir-dat74"))
2105                 host->pwr_reg_add |= MCI_ST_DATA74DIREN;
2106         if (of_property_read_bool(np, "st,sig-dir-cmd"))
2107                 host->pwr_reg_add |= MCI_ST_CMDDIREN;
2108         if (of_property_read_bool(np, "st,sig-pin-fbclk"))
2109                 host->pwr_reg_add |= MCI_ST_FBCLKEN;
2110         if (of_property_read_bool(np, "st,sig-dir"))
2111                 host->pwr_reg_add |= MCI_STM32_DIRPOL;
2112         if (of_property_read_bool(np, "st,neg-edge"))
2113                 host->clk_reg_add |= MCI_STM32_CLK_NEGEDGE;
2114         if (of_property_read_bool(np, "st,use-ckin"))
2115                 mmci_probe_level_translator(mmc);
2116
2117         if (of_property_read_bool(np, "mmc-cap-mmc-highspeed"))
2118                 mmc->caps |= MMC_CAP_MMC_HIGHSPEED;
2119         if (of_property_read_bool(np, "mmc-cap-sd-highspeed"))
2120                 mmc->caps |= MMC_CAP_SD_HIGHSPEED;
2121
2122         return 0;
2123 }
2124
2125 static int mmci_probe(struct amba_device *dev,
2126         const struct amba_id *id)
2127 {
2128         struct mmci_platform_data *plat = dev->dev.platform_data;
2129         struct device_node *np = dev->dev.of_node;
2130         struct variant_data *variant = id->data;
2131         struct mmci_host *host;
2132         struct mmc_host *mmc;
2133         int ret;
2134
2135         /* Must have platform data or Device Tree. */
2136         if (!plat && !np) {
2137                 dev_err(&dev->dev, "No plat data or DT found\n");
2138                 return -EINVAL;
2139         }
2140
2141         if (!plat) {
2142                 plat = devm_kzalloc(&dev->dev, sizeof(*plat), GFP_KERNEL);
2143                 if (!plat)
2144                         return -ENOMEM;
2145         }
2146
2147         mmc = mmc_alloc_host(sizeof(struct mmci_host), &dev->dev);
2148         if (!mmc)
2149                 return -ENOMEM;
2150
2151         host = mmc_priv(mmc);
2152         host->mmc = mmc;
2153         host->mmc_ops = &mmci_ops;
2154         mmc->ops = &mmci_ops;
2155
2156         ret = mmci_of_parse(np, mmc);
2157         if (ret)
2158                 goto host_free;
2159
2160         /*
2161          * Some variant (STM32) doesn't have opendrain bit, nevertheless
2162          * pins can be set accordingly using pinctrl
2163          */
2164         if (!variant->opendrain) {
2165                 host->pinctrl = devm_pinctrl_get(&dev->dev);
2166                 if (IS_ERR(host->pinctrl)) {
2167                         dev_err(&dev->dev, "failed to get pinctrl");
2168                         ret = PTR_ERR(host->pinctrl);
2169                         goto host_free;
2170                 }
2171
2172                 host->pins_opendrain = pinctrl_lookup_state(host->pinctrl,
2173                                                             MMCI_PINCTRL_STATE_OPENDRAIN);
2174                 if (IS_ERR(host->pins_opendrain)) {
2175                         dev_err(mmc_dev(mmc), "Can't select opendrain pins\n");
2176                         ret = PTR_ERR(host->pins_opendrain);
2177                         goto host_free;
2178                 }
2179         }
2180
2181         host->hw_designer = amba_manf(dev);
2182         host->hw_revision = amba_rev(dev);
2183         dev_dbg(mmc_dev(mmc), "designer ID = 0x%02x\n", host->hw_designer);
2184         dev_dbg(mmc_dev(mmc), "revision = 0x%01x\n", host->hw_revision);
2185
2186         host->clk = devm_clk_get(&dev->dev, NULL);
2187         if (IS_ERR(host->clk)) {
2188                 ret = PTR_ERR(host->clk);
2189                 goto host_free;
2190         }
2191
2192         ret = clk_prepare_enable(host->clk);
2193         if (ret)
2194                 goto host_free;
2195
2196         if (variant->qcom_fifo)
2197                 host->get_rx_fifocnt = mmci_qcom_get_rx_fifocnt;
2198         else
2199                 host->get_rx_fifocnt = mmci_get_rx_fifocnt;
2200
2201         host->plat = plat;
2202         host->variant = variant;
2203         host->mclk = clk_get_rate(host->clk);
2204         /*
2205          * According to the spec, mclk is max 100 MHz,
2206          * so we try to adjust the clock down to this,
2207          * (if possible).
2208          */
2209         if (host->mclk > variant->f_max) {
2210                 ret = clk_set_rate(host->clk, variant->f_max);
2211                 if (ret < 0)
2212                         goto clk_disable;
2213                 host->mclk = clk_get_rate(host->clk);
2214                 dev_dbg(mmc_dev(mmc), "eventual mclk rate: %u Hz\n",
2215                         host->mclk);
2216         }
2217
2218         host->phybase = dev->res.start;
2219         host->base = devm_ioremap_resource(&dev->dev, &dev->res);
2220         if (IS_ERR(host->base)) {
2221                 ret = PTR_ERR(host->base);
2222                 goto clk_disable;
2223         }
2224
2225         if (variant->init)
2226                 variant->init(host);
2227
2228         /*
2229          * The ARM and ST versions of the block have slightly different
2230          * clock divider equations which means that the minimum divider
2231          * differs too.
2232          * on Qualcomm like controllers get the nearest minimum clock to 100Khz
2233          */
2234         if (variant->st_clkdiv)
2235                 mmc->f_min = DIV_ROUND_UP(host->mclk, 257);
2236         else if (variant->stm32_clkdiv)
2237                 mmc->f_min = DIV_ROUND_UP(host->mclk, 2046);
2238         else if (variant->explicit_mclk_control)
2239                 mmc->f_min = clk_round_rate(host->clk, 100000);
2240         else
2241                 mmc->f_min = DIV_ROUND_UP(host->mclk, 512);
2242         /*
2243          * If no maximum operating frequency is supplied, fall back to use
2244          * the module parameter, which has a (low) default value in case it
2245          * is not specified. Either value must not exceed the clock rate into
2246          * the block, of course.
2247          */
2248         if (mmc->f_max)
2249                 mmc->f_max = variant->explicit_mclk_control ?
2250                                 min(variant->f_max, mmc->f_max) :
2251                                 min(host->mclk, mmc->f_max);
2252         else
2253                 mmc->f_max = variant->explicit_mclk_control ?
2254                                 fmax : min(host->mclk, fmax);
2255
2256
2257         dev_dbg(mmc_dev(mmc), "clocking block at %u Hz\n", mmc->f_max);
2258
2259         host->rst = devm_reset_control_get_optional_exclusive(&dev->dev, NULL);
2260         if (IS_ERR(host->rst)) {
2261                 ret = PTR_ERR(host->rst);
2262                 goto clk_disable;
2263         }
2264         ret = reset_control_deassert(host->rst);
2265         if (ret)
2266                 dev_err(mmc_dev(mmc), "failed to de-assert reset\n");
2267
2268         /* Get regulators and the supported OCR mask */
2269         ret = mmc_regulator_get_supply(mmc);
2270         if (ret)
2271                 goto clk_disable;
2272
2273         if (!mmc->ocr_avail)
2274                 mmc->ocr_avail = plat->ocr_mask;
2275         else if (plat->ocr_mask)
2276                 dev_warn(mmc_dev(mmc), "Platform OCR mask is ignored\n");
2277
2278         /* We support these capabilities. */
2279         mmc->caps |= MMC_CAP_CMD23;
2280
2281         /*
2282          * Enable busy detection.
2283          */
2284         if (variant->busy_detect) {
2285                 mmci_ops.card_busy = mmci_card_busy;
2286                 /*
2287                  * Not all variants have a flag to enable busy detection
2288                  * in the DPSM, but if they do, set it here.
2289                  */
2290                 if (variant->busy_dpsm_flag)
2291                         mmci_write_datactrlreg(host,
2292                                                host->variant->busy_dpsm_flag);
2293                 mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
2294         }
2295
2296         /* Variants with mandatory busy timeout in HW needs R1B responses. */
2297         if (variant->busy_timeout)
2298                 mmc->caps |= MMC_CAP_NEED_RSP_BUSY;
2299
2300         /* Prepare a CMD12 - needed to clear the DPSM on some variants. */
2301         host->stop_abort.opcode = MMC_STOP_TRANSMISSION;
2302         host->stop_abort.arg = 0;
2303         host->stop_abort.flags = MMC_RSP_R1B | MMC_CMD_AC;
2304
2305         /* We support these PM capabilities. */
2306         mmc->pm_caps |= MMC_PM_KEEP_POWER;
2307
2308         /*
2309          * We can do SGIO
2310          */
2311         mmc->max_segs = NR_SG;
2312
2313         /*
2314          * Since only a certain number of bits are valid in the data length
2315          * register, we must ensure that we don't exceed 2^num-1 bytes in a
2316          * single request.
2317          */
2318         mmc->max_req_size = (1 << variant->datalength_bits) - 1;
2319
2320         /*
2321          * Set the maximum segment size.  Since we aren't doing DMA
2322          * (yet) we are only limited by the data length register.
2323          */
2324         mmc->max_seg_size = mmc->max_req_size;
2325
2326         /*
2327          * Block size can be up to 2048 bytes, but must be a power of two.
2328          */
2329         mmc->max_blk_size = 1 << variant->datactrl_blocksz;
2330
2331         /*
2332          * Limit the number of blocks transferred so that we don't overflow
2333          * the maximum request size.
2334          */
2335         mmc->max_blk_count = mmc->max_req_size >> variant->datactrl_blocksz;
2336
2337         spin_lock_init(&host->lock);
2338
2339         writel(0, host->base + MMCIMASK0);
2340
2341         if (variant->mmcimask1)
2342                 writel(0, host->base + MMCIMASK1);
2343
2344         writel(0xfff, host->base + MMCICLEAR);
2345
2346         /*
2347          * If:
2348          * - not using DT but using a descriptor table, or
2349          * - using a table of descriptors ALONGSIDE DT, or
2350          * look up these descriptors named "cd" and "wp" right here, fail
2351          * silently of these do not exist
2352          */
2353         if (!np) {
2354                 ret = mmc_gpiod_request_cd(mmc, "cd", 0, false, 0);
2355                 if (ret == -EPROBE_DEFER)
2356                         goto clk_disable;
2357
2358                 ret = mmc_gpiod_request_ro(mmc, "wp", 0, 0);
2359                 if (ret == -EPROBE_DEFER)
2360                         goto clk_disable;
2361         }
2362
2363         ret = devm_request_threaded_irq(&dev->dev, dev->irq[0], mmci_irq,
2364                                         mmci_irq_thread, IRQF_SHARED,
2365                                         DRIVER_NAME " (cmd)", host);
2366         if (ret)
2367                 goto clk_disable;
2368
2369         if (!dev->irq[1])
2370                 host->singleirq = true;
2371         else {
2372                 ret = devm_request_irq(&dev->dev, dev->irq[1], mmci_pio_irq,
2373                                 IRQF_SHARED, DRIVER_NAME " (pio)", host);
2374                 if (ret)
2375                         goto clk_disable;
2376         }
2377
2378         if (host->variant->busy_detect)
2379                 INIT_DELAYED_WORK(&host->ux500_busy_timeout_work,
2380                                   ux500_busy_timeout_work);
2381
2382         writel(MCI_IRQENABLE | variant->start_err, host->base + MMCIMASK0);
2383
2384         amba_set_drvdata(dev, mmc);
2385
2386         dev_info(&dev->dev, "%s: PL%03x manf %x rev%u at 0x%08llx irq %d,%d (pio)\n",
2387                  mmc_hostname(mmc), amba_part(dev), amba_manf(dev),
2388                  amba_rev(dev), (unsigned long long)dev->res.start,
2389                  dev->irq[0], dev->irq[1]);
2390
2391         mmci_dma_setup(host);
2392
2393         pm_runtime_set_autosuspend_delay(&dev->dev, 50);
2394         pm_runtime_use_autosuspend(&dev->dev);
2395
2396         ret = mmc_add_host(mmc);
2397         if (ret)
2398                 goto clk_disable;
2399
2400         pm_runtime_put(&dev->dev);
2401         return 0;
2402
2403  clk_disable:
2404         clk_disable_unprepare(host->clk);
2405  host_free:
2406         mmc_free_host(mmc);
2407         return ret;
2408 }
2409
2410 static void mmci_remove(struct amba_device *dev)
2411 {
2412         struct mmc_host *mmc = amba_get_drvdata(dev);
2413
2414         if (mmc) {
2415                 struct mmci_host *host = mmc_priv(mmc);
2416                 struct variant_data *variant = host->variant;
2417
2418                 /*
2419                  * Undo pm_runtime_put() in probe.  We use the _sync
2420                  * version here so that we can access the primecell.
2421                  */
2422                 pm_runtime_get_sync(&dev->dev);
2423
2424                 mmc_remove_host(mmc);
2425
2426                 writel(0, host->base + MMCIMASK0);
2427
2428                 if (variant->mmcimask1)
2429                         writel(0, host->base + MMCIMASK1);
2430
2431                 writel(0, host->base + MMCICOMMAND);
2432                 writel(0, host->base + MMCIDATACTRL);
2433
2434                 mmci_dma_release(host);
2435                 clk_disable_unprepare(host->clk);
2436                 mmc_free_host(mmc);
2437         }
2438 }
2439
2440 #ifdef CONFIG_PM
2441 static void mmci_save(struct mmci_host *host)
2442 {
2443         unsigned long flags;
2444
2445         spin_lock_irqsave(&host->lock, flags);
2446
2447         writel(0, host->base + MMCIMASK0);
2448         if (host->variant->pwrreg_nopower) {
2449                 writel(0, host->base + MMCIDATACTRL);
2450                 writel(0, host->base + MMCIPOWER);
2451                 writel(0, host->base + MMCICLOCK);
2452         }
2453         mmci_reg_delay(host);
2454
2455         spin_unlock_irqrestore(&host->lock, flags);
2456 }
2457
2458 static void mmci_restore(struct mmci_host *host)
2459 {
2460         unsigned long flags;
2461
2462         spin_lock_irqsave(&host->lock, flags);
2463
2464         if (host->variant->pwrreg_nopower) {
2465                 writel(host->clk_reg, host->base + MMCICLOCK);
2466                 writel(host->datactrl_reg, host->base + MMCIDATACTRL);
2467                 writel(host->pwr_reg, host->base + MMCIPOWER);
2468         }
2469         writel(MCI_IRQENABLE | host->variant->start_err,
2470                host->base + MMCIMASK0);
2471         mmci_reg_delay(host);
2472
2473         spin_unlock_irqrestore(&host->lock, flags);
2474 }
2475
2476 static int mmci_runtime_suspend(struct device *dev)
2477 {
2478         struct amba_device *adev = to_amba_device(dev);
2479         struct mmc_host *mmc = amba_get_drvdata(adev);
2480
2481         if (mmc) {
2482                 struct mmci_host *host = mmc_priv(mmc);
2483                 pinctrl_pm_select_sleep_state(dev);
2484                 mmci_save(host);
2485                 clk_disable_unprepare(host->clk);
2486         }
2487
2488         return 0;
2489 }
2490
2491 static int mmci_runtime_resume(struct device *dev)
2492 {
2493         struct amba_device *adev = to_amba_device(dev);
2494         struct mmc_host *mmc = amba_get_drvdata(adev);
2495
2496         if (mmc) {
2497                 struct mmci_host *host = mmc_priv(mmc);
2498                 clk_prepare_enable(host->clk);
2499                 mmci_restore(host);
2500                 pinctrl_select_default_state(dev);
2501         }
2502
2503         return 0;
2504 }
2505 #endif
2506
2507 static const struct dev_pm_ops mmci_dev_pm_ops = {
2508         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2509                                 pm_runtime_force_resume)
2510         SET_RUNTIME_PM_OPS(mmci_runtime_suspend, mmci_runtime_resume, NULL)
2511 };
2512
2513 static const struct amba_id mmci_ids[] = {
2514         {
2515                 .id     = 0x00041180,
2516                 .mask   = 0xff0fffff,
2517                 .data   = &variant_arm,
2518         },
2519         {
2520                 .id     = 0x01041180,
2521                 .mask   = 0xff0fffff,
2522                 .data   = &variant_arm_extended_fifo,
2523         },
2524         {
2525                 .id     = 0x02041180,
2526                 .mask   = 0xff0fffff,
2527                 .data   = &variant_arm_extended_fifo_hwfc,
2528         },
2529         {
2530                 .id     = 0x00041181,
2531                 .mask   = 0x000fffff,
2532                 .data   = &variant_arm,
2533         },
2534         /* ST Micro variants */
2535         {
2536                 .id     = 0x00180180,
2537                 .mask   = 0x00ffffff,
2538                 .data   = &variant_u300,
2539         },
2540         {
2541                 .id     = 0x10180180,
2542                 .mask   = 0xf0ffffff,
2543                 .data   = &variant_nomadik,
2544         },
2545         {
2546                 .id     = 0x00280180,
2547                 .mask   = 0x00ffffff,
2548                 .data   = &variant_nomadik,
2549         },
2550         {
2551                 .id     = 0x00480180,
2552                 .mask   = 0xf0ffffff,
2553                 .data   = &variant_ux500,
2554         },
2555         {
2556                 .id     = 0x10480180,
2557                 .mask   = 0xf0ffffff,
2558                 .data   = &variant_ux500v2,
2559         },
2560         {
2561                 .id     = 0x00880180,
2562                 .mask   = 0x00ffffff,
2563                 .data   = &variant_stm32,
2564         },
2565         {
2566                 .id     = 0x10153180,
2567                 .mask   = 0xf0ffffff,
2568                 .data   = &variant_stm32_sdmmc,
2569         },
2570         {
2571                 .id     = 0x00253180,
2572                 .mask   = 0xf0ffffff,
2573                 .data   = &variant_stm32_sdmmcv2,
2574         },
2575         {
2576                 .id     = 0x20253180,
2577                 .mask   = 0xf0ffffff,
2578                 .data   = &variant_stm32_sdmmcv2,
2579         },
2580         {
2581                 .id     = 0x00353180,
2582                 .mask   = 0xf0ffffff,
2583                 .data   = &variant_stm32_sdmmcv3,
2584         },
2585         /* Qualcomm variants */
2586         {
2587                 .id     = 0x00051180,
2588                 .mask   = 0x000fffff,
2589                 .data   = &variant_qcom,
2590         },
2591         { 0, 0 },
2592 };
2593
2594 MODULE_DEVICE_TABLE(amba, mmci_ids);
2595
2596 static struct amba_driver mmci_driver = {
2597         .drv            = {
2598                 .name   = DRIVER_NAME,
2599                 .pm     = &mmci_dev_pm_ops,
2600                 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
2601         },
2602         .probe          = mmci_probe,
2603         .remove         = mmci_remove,
2604         .id_table       = mmci_ids,
2605 };
2606
2607 module_amba_driver(mmci_driver);
2608
2609 module_param(fmax, uint, 0444);
2610
2611 MODULE_DESCRIPTION("ARM PrimeCell PL180/181 Multimedia Card Interface driver");
2612 MODULE_LICENSE("GPL");