drivers/mmc/host/sdhci-tegra.c

   1 /*
   2  * Copyright (C) 2010 Google, Inc.
   3  *
   4  * This software is licensed under the terms of the GNU General Public
   5  * License version 2, as published by the Free Software Foundation, and
   6  * may be copied, distributed, and modified under those terms.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  */
  14
  15 #include <linux/delay.h>
  16 #include <linux/err.h>
  17 #include <linux/module.h>
  18 #include <linux/init.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/clk.h>
  22 #include <linux/io.h>
  23 #include <linux/of.h>
  24 #include <linux/of_device.h>
  25 #include <linux/pinctrl/consumer.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/reset.h>
  28 #include <linux/mmc/card.h>
  29 #include <linux/mmc/host.h>
  30 #include <linux/mmc/mmc.h>
  31 #include <linux/mmc/slot-gpio.h>
  32 #include <linux/gpio/consumer.h>
  33 #include <linux/ktime.h>
  34
  35 #include "sdhci-pltfm.h"
  36 #include "cqhci.h"
  37
  38 /* Tegra SDHOST controller vendor register definitions */
  39 #define SDHCI_TEGRA_VENDOR_CLOCK_CTRL                   0x100
  40 #define SDHCI_CLOCK_CTRL_TAP_MASK                       0x00ff0000
  41 #define SDHCI_CLOCK_CTRL_TAP_SHIFT                      16
  42 #define SDHCI_CLOCK_CTRL_TRIM_MASK                      0x1f000000
  43 #define SDHCI_CLOCK_CTRL_TRIM_SHIFT                     24
  44 #define SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE          BIT(5)
  45 #define SDHCI_CLOCK_CTRL_PADPIPE_CLKEN_OVERRIDE         BIT(3)
  46 #define SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE        BIT(2)
  47
  48 #define SDHCI_TEGRA_VENDOR_SYS_SW_CTRL                  0x104
  49 #define SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE         BIT(31)
  50
  51 #define SDHCI_TEGRA_VENDOR_CAP_OVERRIDES                0x10c
  52 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK         0x00003f00
  53 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT        8
  54
  55 #define SDHCI_TEGRA_VENDOR_MISC_CTRL                    0x120
  56 #define SDHCI_MISC_CTRL_ENABLE_SDR104                   0x8
  57 #define SDHCI_MISC_CTRL_ENABLE_SDR50                    0x10
  58 #define SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300           0x20
  59 #define SDHCI_MISC_CTRL_ENABLE_DDR50                    0x200
  60
  61 #define SDHCI_TEGRA_VENDOR_DLLCAL_CFG                   0x1b0
  62 #define SDHCI_TEGRA_DLLCAL_CALIBRATE                    BIT(31)
  63
  64 #define SDHCI_TEGRA_VENDOR_DLLCAL_STA                   0x1bc
  65 #define SDHCI_TEGRA_DLLCAL_STA_ACTIVE                   BIT(31)
  66
  67 #define SDHCI_VNDR_TUN_CTRL0_0                          0x1c0
  68 #define SDHCI_VNDR_TUN_CTRL0_TUN_HW_TAP                 0x20000
  69 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK         0x03fc0000
  70 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT        18
  71 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK                 0x00001fc0
  72 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT                6
  73 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK              0x000e000
  74 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT             13
  75 #define TRIES_128                                       2
  76 #define TRIES_256                                       4
  77 #define SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK          0x7
  78
  79 #define SDHCI_TEGRA_VNDR_TUN_CTRL1_0                    0x1c4
  80 #define SDHCI_TEGRA_VNDR_TUN_STATUS0                    0x1C8
  81 #define SDHCI_TEGRA_VNDR_TUN_STATUS1                    0x1CC
  82 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK           0xFF
  83 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT      0x8
  84 #define TUNING_WORD_BIT_SIZE                            32
  85
  86 #define SDHCI_TEGRA_AUTO_CAL_CONFIG                     0x1e4
  87 #define SDHCI_AUTO_CAL_START                            BIT(31)
  88 #define SDHCI_AUTO_CAL_ENABLE                           BIT(29)
  89 #define SDHCI_AUTO_CAL_PDPU_OFFSET_MASK                 0x0000ffff
  90
  91 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL                  0x1e0
  92 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK    0x0000000f
  93 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL     0x7
  94 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD   BIT(31)
  95 #define SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK          0x07FFF000
  96
  97 #define SDHCI_TEGRA_AUTO_CAL_STATUS                     0x1ec
  98 #define SDHCI_TEGRA_AUTO_CAL_ACTIVE                     BIT(31)
  99
 100 #define NVQUIRK_FORCE_SDHCI_SPEC_200                    BIT(0)
 101 #define NVQUIRK_ENABLE_BLOCK_GAP_DET                    BIT(1)
 102 #define NVQUIRK_ENABLE_SDHCI_SPEC_300                   BIT(2)
 103 #define NVQUIRK_ENABLE_SDR50                            BIT(3)
 104 #define NVQUIRK_ENABLE_SDR104                           BIT(4)
 105 #define NVQUIRK_ENABLE_DDR50                            BIT(5)
 106 #define NVQUIRK_HAS_PADCALIB                            BIT(6)
 107 #define NVQUIRK_NEEDS_PAD_CONTROL                       BIT(7)
 108 #define NVQUIRK_DIS_CARD_CLK_CONFIG_TAP                 BIT(8)
 109 #define NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING               BIT(9)
 110
 111 /* SDMMC CQE Base Address for Tegra Host Ver 4.1 and Higher */
 112 #define SDHCI_TEGRA_CQE_BASE_ADDR                       0xF000
 113
 114 struct sdhci_tegra_soc_data {
 115         const struct sdhci_pltfm_data *pdata;
 116         u32 nvquirks;
 117         u8 min_tap_delay;
 118         u8 max_tap_delay;
 119 };
 120
 121 /* Magic pull up and pull down pad calibration offsets */
 122 struct sdhci_tegra_autocal_offsets {
 123         u32 pull_up_3v3;
 124         u32 pull_down_3v3;
 125         u32 pull_up_3v3_timeout;
 126         u32 pull_down_3v3_timeout;
 127         u32 pull_up_1v8;
 128         u32 pull_down_1v8;
 129         u32 pull_up_1v8_timeout;
 130         u32 pull_down_1v8_timeout;
 131         u32 pull_up_sdr104;
 132         u32 pull_down_sdr104;
 133         u32 pull_up_hs400;
 134         u32 pull_down_hs400;
 135 };
 136
 137 struct sdhci_tegra {
 138         const struct sdhci_tegra_soc_data *soc_data;
 139         struct gpio_desc *power_gpio;
 140         bool ddr_signaling;
 141         bool pad_calib_required;
 142         bool pad_control_available;
 143
 144         struct reset_control *rst;
 145         struct pinctrl *pinctrl_sdmmc;
 146         struct pinctrl_state *pinctrl_state_3v3;
 147         struct pinctrl_state *pinctrl_state_1v8;
 148         struct pinctrl_state *pinctrl_state_3v3_drv;
 149         struct pinctrl_state *pinctrl_state_1v8_drv;
 150
 151         struct sdhci_tegra_autocal_offsets autocal_offsets;
 152         ktime_t last_calib;
 153
 154         u32 default_tap;
 155         u32 default_trim;
 156         u32 dqs_trim;
 157         bool enable_hwcq;
 158         unsigned long curr_clk_rate;
 159         u8 tuned_tap_delay;
 160 };
 161
 162 static u16 tegra_sdhci_readw(struct sdhci_host *host, int reg)
 163 {
 164         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 165         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 166         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 167
 168         if (unlikely((soc_data->nvquirks & NVQUIRK_FORCE_SDHCI_SPEC_200) &&
 169                         (reg == SDHCI_HOST_VERSION))) {
 170                 /* Erratum: Version register is invalid in HW. */
 171                 return SDHCI_SPEC_200;
 172         }
 173
 174         return readw(host->ioaddr + reg);
 175 }
 176
 177 static void tegra_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 178 {
 179         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 180
 181         switch (reg) {
 182         case SDHCI_TRANSFER_MODE:
 183                 /*
 184                  * Postpone this write, we must do it together with a
 185                  * command write that is down below.
 186                  */
 187                 pltfm_host->xfer_mode_shadow = val;
 188                 return;
 189         case SDHCI_COMMAND:
 190                 writel((val << 16) | pltfm_host->xfer_mode_shadow,
 191                         host->ioaddr + SDHCI_TRANSFER_MODE);
 192                 return;
 193         }
 194
 195         writew(val, host->ioaddr + reg);
 196 }
 197
 198 static void tegra_sdhci_writel(struct sdhci_host *host, u32 val, int reg)
 199 {
 200         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 201         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 202         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 203
 204         /* Seems like we're getting spurious timeout and crc errors, so
 205          * disable signalling of them. In case of real errors software
 206          * timers should take care of eventually detecting them.
 207          */
 208         if (unlikely(reg == SDHCI_SIGNAL_ENABLE))
 209                 val &= ~(SDHCI_INT_TIMEOUT|SDHCI_INT_CRC);
 210
 211         writel(val, host->ioaddr + reg);
 212
 213         if (unlikely((soc_data->nvquirks & NVQUIRK_ENABLE_BLOCK_GAP_DET) &&
 214                         (reg == SDHCI_INT_ENABLE))) {
 215                 /* Erratum: Must enable block gap interrupt detection */
 216                 u8 gap_ctrl = readb(host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 217                 if (val & SDHCI_INT_CARD_INT)
 218                         gap_ctrl |= 0x8;
 219                 else
 220                         gap_ctrl &= ~0x8;
 221                 writeb(gap_ctrl, host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 222         }
 223 }
 224
 225 static bool tegra_sdhci_configure_card_clk(struct sdhci_host *host, bool enable)
 226 {
 227         bool status;
 228         u32 reg;
 229
 230         reg = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
 231         status = !!(reg & SDHCI_CLOCK_CARD_EN);
 232
 233         if (status == enable)
 234                 return status;
 235
 236         if (enable)
 237                 reg |= SDHCI_CLOCK_CARD_EN;
 238         else
 239                 reg &= ~SDHCI_CLOCK_CARD_EN;
 240
 241         sdhci_writew(host, reg, SDHCI_CLOCK_CONTROL);
 242
 243         return status;
 244 }
 245
 246 static void tegra210_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 247 {
 248         bool is_tuning_cmd = 0;
 249         bool clk_enabled;
 250         u8 cmd;
 251
 252         if (reg == SDHCI_COMMAND) {
 253                 cmd = SDHCI_GET_CMD(val);
 254                 is_tuning_cmd = cmd == MMC_SEND_TUNING_BLOCK ||
 255                                 cmd == MMC_SEND_TUNING_BLOCK_HS200;
 256         }
 257
 258         if (is_tuning_cmd)
 259                 clk_enabled = tegra_sdhci_configure_card_clk(host, 0);
 260
 261         writew(val, host->ioaddr + reg);
 262
 263         if (is_tuning_cmd) {
 264                 udelay(1);
 265                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 266                 tegra_sdhci_configure_card_clk(host, clk_enabled);
 267         }
 268 }
 269
 270 static bool tegra_sdhci_is_pad_and_regulator_valid(struct sdhci_host *host)
 271 {
 272         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 273         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 274         int has_1v8, has_3v3;
 275
 276         /*
 277          * The SoCs which have NVQUIRK_NEEDS_PAD_CONTROL require software pad
 278          * voltage configuration in order to perform voltage switching. This
 279          * means that valid pinctrl info is required on SDHCI instances capable
 280          * of performing voltage switching. Whether or not an SDHCI instance is
 281          * capable of voltage switching is determined based on the regulator.
 282          */
 283
 284         if (!(tegra_host->soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL))
 285                 return true;
 286
 287         if (IS_ERR(host->mmc->supply.vqmmc))
 288                 return false;
 289
 290         has_1v8 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 291                                                  1700000, 1950000);
 292
 293         has_3v3 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 294                                                  2700000, 3600000);
 295
 296         if (has_1v8 == 1 && has_3v3 == 1)
 297                 return tegra_host->pad_control_available;
 298
 299         /* Fixed voltage, no pad control required. */
 300         return true;
 301 }
 302
 303 static void tegra_sdhci_set_tap(struct sdhci_host *host, unsigned int tap)
 304 {
 305         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 306         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 307         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 308         bool card_clk_enabled = false;
 309         u32 reg;
 310
 311         /*
 312          * Touching the tap values is a bit tricky on some SoC generations.
 313          * The quirk enables a workaround for a glitch that sometimes occurs if
 314          * the tap values are changed.
 315          */
 316
 317         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP)
 318                 card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 319
 320         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 321         reg &= ~SDHCI_CLOCK_CTRL_TAP_MASK;
 322         reg |= tap << SDHCI_CLOCK_CTRL_TAP_SHIFT;
 323         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 324
 325         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP &&
 326             card_clk_enabled) {
 327                 udelay(1);
 328                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 329                 tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 330         }
 331 }
 332
 333 static void tegra_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
 334                                               struct mmc_ios *ios)
 335 {
 336         struct sdhci_host *host = mmc_priv(mmc);
 337         u32 val;
 338
 339         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 340
 341         if (ios->enhanced_strobe)
 342                 val |= SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 343         else
 344                 val &= ~SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 345
 346         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 347
 348 }
 349
 350 static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask)
 351 {
 352         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 353         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 354         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 355         u32 misc_ctrl, clk_ctrl, pad_ctrl;
 356
 357         sdhci_reset(host, mask);
 358
 359         if (!(mask & SDHCI_RESET_ALL))
 360                 return;
 361
 362         tegra_sdhci_set_tap(host, tegra_host->default_tap);
 363
 364         misc_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 365         clk_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 366
 367         misc_ctrl &= ~(SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300 |
 368                        SDHCI_MISC_CTRL_ENABLE_SDR50 |
 369                        SDHCI_MISC_CTRL_ENABLE_DDR50 |
 370                        SDHCI_MISC_CTRL_ENABLE_SDR104);
 371
 372         clk_ctrl &= ~(SDHCI_CLOCK_CTRL_TRIM_MASK |
 373                       SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE);
 374
 375         if (tegra_sdhci_is_pad_and_regulator_valid(host)) {
 376                 /* Erratum: Enable SDHCI spec v3.00 support */
 377                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDHCI_SPEC_300)
 378                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300;
 379                 /* Advertise UHS modes as supported by host */
 380                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR50)
 381                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR50;
 382                 if (soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
 383                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_DDR50;
 384                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR104)
 385                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR104;
 386                 if (soc_data->nvquirks & SDHCI_MISC_CTRL_ENABLE_SDR50)
 387                         clk_ctrl |= SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE;
 388         }
 389
 390         clk_ctrl |= tegra_host->default_trim << SDHCI_CLOCK_CTRL_TRIM_SHIFT;
 391
 392         sdhci_writel(host, misc_ctrl, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 393         sdhci_writel(host, clk_ctrl, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 394
 395         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB) {
 396                 pad_ctrl = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 397                 pad_ctrl &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK;
 398                 pad_ctrl |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL;
 399                 sdhci_writel(host, pad_ctrl, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 400
 401                 tegra_host->pad_calib_required = true;
 402         }
 403
 404         tegra_host->ddr_signaling = false;
 405 }
 406
 407 static void tegra_sdhci_configure_cal_pad(struct sdhci_host *host, bool enable)
 408 {
 409         u32 val;
 410
 411         /*
 412          * Enable or disable the additional I/O pad used by the drive strength
 413          * calibration process.
 414          */
 415         val = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 416
 417         if (enable)
 418                 val |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 419         else
 420                 val &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 421
 422         sdhci_writel(host, val, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 423
 424         if (enable)
 425                 usleep_range(1, 2);
 426 }
 427
 428 static void tegra_sdhci_set_pad_autocal_offset(struct sdhci_host *host,
 429                                                u16 pdpu)
 430 {
 431         u32 reg;
 432
 433         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 434         reg &= ~SDHCI_AUTO_CAL_PDPU_OFFSET_MASK;
 435         reg |= pdpu;
 436         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 437 }
 438
 439 static int tegra_sdhci_set_padctrl(struct sdhci_host *host, int voltage,
 440                                    bool state_drvupdn)
 441 {
 442         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 443         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 444         struct sdhci_tegra_autocal_offsets *offsets =
 445                                                 &tegra_host->autocal_offsets;
 446         struct pinctrl_state *pinctrl_drvupdn = NULL;
 447         int ret = 0;
 448         u8 drvup = 0, drvdn = 0;
 449         u32 reg;
 450
 451         if (!state_drvupdn) {
 452                 /* PADS Drive Strength */
 453                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 454                         if (tegra_host->pinctrl_state_1v8_drv) {
 455                                 pinctrl_drvupdn =
 456                                         tegra_host->pinctrl_state_1v8_drv;
 457                         } else {
 458                                 drvup = offsets->pull_up_1v8_timeout;
 459                                 drvdn = offsets->pull_down_1v8_timeout;
 460                         }
 461                 } else {
 462                         if (tegra_host->pinctrl_state_3v3_drv) {
 463                                 pinctrl_drvupdn =
 464                                         tegra_host->pinctrl_state_3v3_drv;
 465                         } else {
 466                                 drvup = offsets->pull_up_3v3_timeout;
 467                                 drvdn = offsets->pull_down_3v3_timeout;
 468                         }
 469                 }
 470
 471                 if (pinctrl_drvupdn != NULL) {
 472                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 473                                                         pinctrl_drvupdn);
 474                         if (ret < 0)
 475                                 dev_err(mmc_dev(host->mmc),
 476                                         "failed pads drvupdn, ret: %d\n", ret);
 477                 } else if ((drvup) || (drvdn)) {
 478                         reg = sdhci_readl(host,
 479                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 480                         reg &= ~SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK;
 481                         reg |= (drvup << 20) | (drvdn << 12);
 482                         sdhci_writel(host, reg,
 483                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 484                 }
 485
 486         } else {
 487                 /* Dual Voltage PADS Voltage selection */
 488                 if (!tegra_host->pad_control_available)
 489                         return 0;
 490
 491                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 492                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 493                                                 tegra_host->pinctrl_state_1v8);
 494                         if (ret < 0)
 495                                 dev_err(mmc_dev(host->mmc),
 496                                         "setting 1.8V failed, ret: %d\n", ret);
 497                 } else {
 498                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 499                                                 tegra_host->pinctrl_state_3v3);
 500                         if (ret < 0)
 501                                 dev_err(mmc_dev(host->mmc),
 502                                         "setting 3.3V failed, ret: %d\n", ret);
 503                 }
 504         }
 505
 506         return ret;
 507 }
 508
 509 static void tegra_sdhci_pad_autocalib(struct sdhci_host *host)
 510 {
 511         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 512         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 513         struct sdhci_tegra_autocal_offsets offsets =
 514                         tegra_host->autocal_offsets;
 515         struct mmc_ios *ios = &host->mmc->ios;
 516         bool card_clk_enabled;
 517         u16 pdpu;
 518         u32 reg;
 519         int ret;
 520
 521         switch (ios->timing) {
 522         case MMC_TIMING_UHS_SDR104:
 523                 pdpu = offsets.pull_down_sdr104 << 8 | offsets.pull_up_sdr104;
 524                 break;
 525         case MMC_TIMING_MMC_HS400:
 526                 pdpu = offsets.pull_down_hs400 << 8 | offsets.pull_up_hs400;
 527                 break;
 528         default:
 529                 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
 530                         pdpu = offsets.pull_down_1v8 << 8 | offsets.pull_up_1v8;
 531                 else
 532                         pdpu = offsets.pull_down_3v3 << 8 | offsets.pull_up_3v3;
 533         }
 534
 535         /* Set initial offset before auto-calibration */
 536         tegra_sdhci_set_pad_autocal_offset(host, pdpu);
 537
 538         card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 539
 540         tegra_sdhci_configure_cal_pad(host, true);
 541
 542         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 543         reg |= SDHCI_AUTO_CAL_ENABLE | SDHCI_AUTO_CAL_START;
 544         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 545
 546         usleep_range(1, 2);
 547         /* 10 ms timeout */
 548         ret = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_AUTO_CAL_STATUS,
 549                                  reg, !(reg & SDHCI_TEGRA_AUTO_CAL_ACTIVE),
 550                                  1000, 10000);
 551
 552         tegra_sdhci_configure_cal_pad(host, false);
 553
 554         tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 555
 556         if (ret) {
 557                 dev_err(mmc_dev(host->mmc), "Pad autocal timed out\n");
 558
 559                 /* Disable automatic cal and use fixed Drive Strengths */
 560                 reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 561                 reg &= ~SDHCI_AUTO_CAL_ENABLE;
 562                 sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 563
 564                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, false);
 565                 if (ret < 0)
 566                         dev_err(mmc_dev(host->mmc),
 567                                 "Setting drive strengths failed: %d\n", ret);
 568         }
 569 }
 570
 571 static void tegra_sdhci_parse_pad_autocal_dt(struct sdhci_host *host)
 572 {
 573         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 574         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 575         struct sdhci_tegra_autocal_offsets *autocal =
 576                         &tegra_host->autocal_offsets;
 577         int err;
 578
 579         err = device_property_read_u32(host->mmc->parent,
 580                         "nvidia,pad-autocal-pull-up-offset-3v3",
 581                         &autocal->pull_up_3v3);
 582         if (err)
 583                 autocal->pull_up_3v3 = 0;
 584
 585         err = device_property_read_u32(host->mmc->parent,
 586                         "nvidia,pad-autocal-pull-down-offset-3v3",
 587                         &autocal->pull_down_3v3);
 588         if (err)
 589                 autocal->pull_down_3v3 = 0;
 590
 591         err = device_property_read_u32(host->mmc->parent,
 592                         "nvidia,pad-autocal-pull-up-offset-1v8",
 593                         &autocal->pull_up_1v8);
 594         if (err)
 595                 autocal->pull_up_1v8 = 0;
 596
 597         err = device_property_read_u32(host->mmc->parent,
 598                         "nvidia,pad-autocal-pull-down-offset-1v8",
 599                         &autocal->pull_down_1v8);
 600         if (err)
 601                 autocal->pull_down_1v8 = 0;
 602
 603         err = device_property_read_u32(host->mmc->parent,
 604                         "nvidia,pad-autocal-pull-up-offset-3v3-timeout",
 605                         &autocal->pull_up_3v3_timeout);
 606         if (err) {
 607                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 608                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 609                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 610                                 mmc_hostname(host->mmc));
 611                 autocal->pull_up_3v3_timeout = 0;
 612         }
 613
 614         err = device_property_read_u32(host->mmc->parent,
 615                         "nvidia,pad-autocal-pull-down-offset-3v3-timeout",
 616                         &autocal->pull_down_3v3_timeout);
 617         if (err) {
 618                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 619                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 620                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 621                                 mmc_hostname(host->mmc));
 622                 autocal->pull_down_3v3_timeout = 0;
 623         }
 624
 625         err = device_property_read_u32(host->mmc->parent,
 626                         "nvidia,pad-autocal-pull-up-offset-1v8-timeout",
 627                         &autocal->pull_up_1v8_timeout);
 628         if (err) {
 629                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 630                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 631                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 632                                 mmc_hostname(host->mmc));
 633                 autocal->pull_up_1v8_timeout = 0;
 634         }
 635
 636         err = device_property_read_u32(host->mmc->parent,
 637                         "nvidia,pad-autocal-pull-down-offset-1v8-timeout",
 638                         &autocal->pull_down_1v8_timeout);
 639         if (err) {
 640                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 641                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 642                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 643                                 mmc_hostname(host->mmc));
 644                 autocal->pull_down_1v8_timeout = 0;
 645         }
 646
 647         err = device_property_read_u32(host->mmc->parent,
 648                         "nvidia,pad-autocal-pull-up-offset-sdr104",
 649                         &autocal->pull_up_sdr104);
 650         if (err)
 651                 autocal->pull_up_sdr104 = autocal->pull_up_1v8;
 652
 653         err = device_property_read_u32(host->mmc->parent,
 654                         "nvidia,pad-autocal-pull-down-offset-sdr104",
 655                         &autocal->pull_down_sdr104);
 656         if (err)
 657                 autocal->pull_down_sdr104 = autocal->pull_down_1v8;
 658
 659         err = device_property_read_u32(host->mmc->parent,
 660                         "nvidia,pad-autocal-pull-up-offset-hs400",
 661                         &autocal->pull_up_hs400);
 662         if (err)
 663                 autocal->pull_up_hs400 = autocal->pull_up_1v8;
 664
 665         err = device_property_read_u32(host->mmc->parent,
 666                         "nvidia,pad-autocal-pull-down-offset-hs400",
 667                         &autocal->pull_down_hs400);
 668         if (err)
 669                 autocal->pull_down_hs400 = autocal->pull_down_1v8;
 670 }
 671
 672 static void tegra_sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 673 {
 674         struct sdhci_host *host = mmc_priv(mmc);
 675         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 676         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 677         ktime_t since_calib = ktime_sub(ktime_get(), tegra_host->last_calib);
 678
 679         /* 100 ms calibration interval is specified in the TRM */
 680         if (ktime_to_ms(since_calib) > 100) {
 681                 tegra_sdhci_pad_autocalib(host);
 682                 tegra_host->last_calib = ktime_get();
 683         }
 684
 685         sdhci_request(mmc, mrq);
 686 }
 687
 688 static void tegra_sdhci_parse_tap_and_trim(struct sdhci_host *host)
 689 {
 690         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 691         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 692         int err;
 693
 694         err = device_property_read_u32(host->mmc->parent, "nvidia,default-tap",
 695                                        &tegra_host->default_tap);
 696         if (err)
 697                 tegra_host->default_tap = 0;
 698
 699         err = device_property_read_u32(host->mmc->parent, "nvidia,default-trim",
 700                                        &tegra_host->default_trim);
 701         if (err)
 702                 tegra_host->default_trim = 0;
 703
 704         err = device_property_read_u32(host->mmc->parent, "nvidia,dqs-trim",
 705                                        &tegra_host->dqs_trim);
 706         if (err)
 707                 tegra_host->dqs_trim = 0x11;
 708 }
 709
 710 static void tegra_sdhci_parse_dt(struct sdhci_host *host)
 711 {
 712         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 713         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 714
 715         if (device_property_read_bool(host->mmc->parent, "supports-cqe"))
 716                 tegra_host->enable_hwcq = true;
 717         else
 718                 tegra_host->enable_hwcq = false;
 719
 720         tegra_sdhci_parse_pad_autocal_dt(host);
 721         tegra_sdhci_parse_tap_and_trim(host);
 722 }
 723
 724 static void tegra_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
 725 {
 726         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 727         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 728         unsigned long host_clk;
 729
 730         if (!clock)
 731                 return sdhci_set_clock(host, clock);
 732
 733         /*
 734          * In DDR50/52 modes the Tegra SDHCI controllers require the SDHCI
 735          * divider to be configured to divided the host clock by two. The SDHCI
 736          * clock divider is calculated as part of sdhci_set_clock() by
 737          * sdhci_calc_clk(). The divider is calculated from host->max_clk and
 738          * the requested clock rate.
 739          *
 740          * By setting the host->max_clk to clock * 2 the divider calculation
 741          * will always result in the correct value for DDR50/52 modes,
 742          * regardless of clock rate rounding, which may happen if the value
 743          * from clk_get_rate() is used.
 744          */
 745         host_clk = tegra_host->ddr_signaling ? clock * 2 : clock;
 746         clk_set_rate(pltfm_host->clk, host_clk);
 747         tegra_host->curr_clk_rate = host_clk;
 748         if (tegra_host->ddr_signaling)
 749                 host->max_clk = host_clk;
 750         else
 751                 host->max_clk = clk_get_rate(pltfm_host->clk);
 752
 753         sdhci_set_clock(host, clock);
 754
 755         if (tegra_host->pad_calib_required) {
 756                 tegra_sdhci_pad_autocalib(host);
 757                 tegra_host->pad_calib_required = false;
 758         }
 759 }
 760
 761 static unsigned int tegra_sdhci_get_max_clock(struct sdhci_host *host)
 762 {
 763         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 764
 765         return clk_round_rate(pltfm_host->clk, UINT_MAX);
 766 }
 767
 768 static void tegra_sdhci_set_dqs_trim(struct sdhci_host *host, u8 trim)
 769 {
 770         u32 val;
 771
 772         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 773         val &= ~SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK;
 774         val |= trim << SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT;
 775         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 776 }
 777
 778 static void tegra_sdhci_hs400_dll_cal(struct sdhci_host *host)
 779 {
 780         u32 reg;
 781         int err;
 782
 783         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 784         reg |= SDHCI_TEGRA_DLLCAL_CALIBRATE;
 785         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 786
 787         /* 1 ms sleep, 5 ms timeout */
 788         err = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_VENDOR_DLLCAL_STA,
 789                                  reg, !(reg & SDHCI_TEGRA_DLLCAL_STA_ACTIVE),
 790                                  1000, 5000);
 791         if (err)
 792                 dev_err(mmc_dev(host->mmc),
 793                         "HS400 delay line calibration timed out\n");
 794 }
 795
 796 static void tegra_sdhci_tap_correction(struct sdhci_host *host, u8 thd_up,
 797                                        u8 thd_low, u8 fixed_tap)
 798 {
 799         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 800         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 801         u32 val, tun_status;
 802         u8 word, bit, edge1, tap, window;
 803         bool tap_result;
 804         bool start_fail = false;
 805         bool start_pass = false;
 806         bool end_pass = false;
 807         bool first_fail = false;
 808         bool first_pass = false;
 809         u8 start_pass_tap = 0;
 810         u8 end_pass_tap = 0;
 811         u8 first_fail_tap = 0;
 812         u8 first_pass_tap = 0;
 813         u8 total_tuning_words = host->tuning_loop_count / TUNING_WORD_BIT_SIZE;
 814
 815         /*
 816          * Read auto-tuned results and extract good valid passing window by
 817          * filtering out un-wanted bubble/partial/merged windows.
 818          */
 819         for (word = 0; word < total_tuning_words; word++) {
 820                 val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
 821                 val &= ~SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK;
 822                 val |= word;
 823                 sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
 824                 tun_status = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS0);
 825                 bit = 0;
 826                 while (bit < TUNING_WORD_BIT_SIZE) {
 827                         tap = word * TUNING_WORD_BIT_SIZE + bit;
 828                         tap_result = tun_status & (1 << bit);
 829                         if (!tap_result && !start_fail) {
 830                                 start_fail = true;
 831                                 if (!first_fail) {
 832                                         first_fail_tap = tap;
 833                                         first_fail = true;
 834                                 }
 835
 836                         } else if (tap_result && start_fail && !start_pass) {
 837                                 start_pass_tap = tap;
 838                                 start_pass = true;
 839                                 if (!first_pass) {
 840                                         first_pass_tap = tap;
 841                                         first_pass = true;
 842                                 }
 843
 844                         } else if (!tap_result && start_fail && start_pass &&
 845                                    !end_pass) {
 846                                 end_pass_tap = tap - 1;
 847                                 end_pass = true;
 848                         } else if (tap_result && start_pass && start_fail &&
 849                                    end_pass) {
 850                                 window = end_pass_tap - start_pass_tap;
 851                                 /* discard merged window and bubble window */
 852                                 if (window >= thd_up || window < thd_low) {
 853                                         start_pass_tap = tap;
 854                                         end_pass = false;
 855                                 } else {
 856                                         /* set tap at middle of valid window */
 857                                         tap = start_pass_tap + window / 2;
 858                                         tegra_host->tuned_tap_delay = tap;
 859                                         return;
 860                                 }
 861                         }
 862
 863                         bit++;
 864                 }
 865         }
 866
 867         if (!first_fail) {
 868                 WARN_ON("no edge detected, continue with hw tuned delay.\n");
 869         } else if (first_pass) {
 870                 /* set tap location at fixed tap relative to the first edge */
 871                 edge1 = first_fail_tap + (first_pass_tap - first_fail_tap) / 2;
 872                 if (edge1 - 1 > fixed_tap)
 873                         tegra_host->tuned_tap_delay = edge1 - fixed_tap;
 874                 else
 875                         tegra_host->tuned_tap_delay = edge1 + fixed_tap;
 876         }
 877 }
 878
 879 static void tegra_sdhci_post_tuning(struct sdhci_host *host)
 880 {
 881         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 882         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 883         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 884         u32 avg_tap_dly, val, min_tap_dly, max_tap_dly;
 885         u8 fixed_tap, start_tap, end_tap, window_width;
 886         u8 thdupper, thdlower;
 887         u8 num_iter;
 888         u32 clk_rate_mhz, period_ps, bestcase, worstcase;
 889
 890         /* retain HW tuned tap to use incase if no correction is needed */
 891         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 892         tegra_host->tuned_tap_delay = (val & SDHCI_CLOCK_CTRL_TAP_MASK) >>
 893                                       SDHCI_CLOCK_CTRL_TAP_SHIFT;
 894         if (soc_data->min_tap_delay && soc_data->max_tap_delay) {
 895                 min_tap_dly = soc_data->min_tap_delay;
 896                 max_tap_dly = soc_data->max_tap_delay;
 897                 clk_rate_mhz = tegra_host->curr_clk_rate / USEC_PER_SEC;
 898                 period_ps = USEC_PER_SEC / clk_rate_mhz;
 899                 bestcase = period_ps / min_tap_dly;
 900                 worstcase = period_ps / max_tap_dly;
 901                 /*
 902                  * Upper and Lower bound thresholds used to detect merged and
 903                  * bubble windows
 904                  */
 905                 thdupper = (2 * worstcase + bestcase) / 2;
 906                 thdlower = worstcase / 4;
 907                 /*
 908                  * fixed tap is used when HW tuning result contains single edge
 909                  * and tap is set at fixed tap delay relative to the first edge
 910                  */
 911                 avg_tap_dly = (period_ps * 2) / (min_tap_dly + max_tap_dly);
 912                 fixed_tap = avg_tap_dly / 2;
 913
 914                 val = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS1);
 915                 start_tap = val & SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 916                 end_tap = (val >> SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT) &
 917                           SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 918                 window_width = end_tap - start_tap;
 919                 num_iter = host->tuning_loop_count;
 920                 /*
 921                  * partial window includes edges of the tuning range.
 922                  * merged window includes more taps so window width is higher
 923                  * than upper threshold.
 924                  */
 925                 if (start_tap == 0 || (end_tap == (num_iter - 1)) ||
 926                     (end_tap == num_iter - 2) || window_width >= thdupper) {
 927                         pr_debug("%s: Apply tuning correction\n",
 928                                  mmc_hostname(host->mmc));
 929                         tegra_sdhci_tap_correction(host, thdupper, thdlower,
 930                                                    fixed_tap);
 931                 }
 932         }
 933
 934         tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
 935 }
 936
 937 static int tegra_sdhci_execute_hw_tuning(struct mmc_host *mmc, u32 opcode)
 938 {
 939         struct sdhci_host *host = mmc_priv(mmc);
 940         int err;
 941
 942         err = sdhci_execute_tuning(mmc, opcode);
 943         if (!err && !host->tuning_err)
 944                 tegra_sdhci_post_tuning(host);
 945
 946         return err;
 947 }
 948
 949 static void tegra_sdhci_set_uhs_signaling(struct sdhci_host *host,
 950                                           unsigned timing)
 951 {
 952         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 953         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 954         bool set_default_tap = false;
 955         bool set_dqs_trim = false;
 956         bool do_hs400_dll_cal = false;
 957         u8 iter = TRIES_256;
 958         u32 val;
 959
 960         tegra_host->ddr_signaling = false;
 961         switch (timing) {
 962         case MMC_TIMING_UHS_SDR50:
 963                 break;
 964         case MMC_TIMING_UHS_SDR104:
 965         case MMC_TIMING_MMC_HS200:
 966                 /* Don't set default tap on tunable modes. */
 967                 iter = TRIES_128;
 968                 break;
 969         case MMC_TIMING_MMC_HS400:
 970                 set_dqs_trim = true;
 971                 do_hs400_dll_cal = true;
 972                 iter = TRIES_128;
 973                 break;
 974         case MMC_TIMING_MMC_DDR52:
 975         case MMC_TIMING_UHS_DDR50:
 976                 tegra_host->ddr_signaling = true;
 977                 set_default_tap = true;
 978                 break;
 979         default:
 980                 set_default_tap = true;
 981                 break;
 982         }
 983
 984         val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
 985         val &= ~(SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK |
 986                  SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK |
 987                  SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK);
 988         val |= (iter << SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT |
 989                 0 << SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT |
 990                 1 << SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT);
 991         sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
 992         sdhci_writel(host, 0, SDHCI_TEGRA_VNDR_TUN_CTRL1_0);
 993
 994         host->tuning_loop_count = (iter == TRIES_128) ? 128 : 256;
 995
 996         sdhci_set_uhs_signaling(host, timing);
 997
 998         tegra_sdhci_pad_autocalib(host);
 999
1000         if (tegra_host->tuned_tap_delay && !set_default_tap)
1001                 tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
1002         else
1003                 tegra_sdhci_set_tap(host, tegra_host->default_tap);
1004
1005         if (set_dqs_trim)
1006                 tegra_sdhci_set_dqs_trim(host, tegra_host->dqs_trim);
1007
1008         if (do_hs400_dll_cal)
1009                 tegra_sdhci_hs400_dll_cal(host);
1010 }
1011
1012 static int tegra_sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
1013 {
1014         unsigned int min, max;
1015
1016         /*
1017          * Start search for minimum tap value at 10, as smaller values are
1018          * may wrongly be reported as working but fail at higher speeds,
1019          * according to the TRM.
1020          */
1021         min = 10;
1022         while (min < 255) {
1023                 tegra_sdhci_set_tap(host, min);
1024                 if (!mmc_send_tuning(host->mmc, opcode, NULL))
1025                         break;
1026                 min++;
1027         }
1028
1029         /* Find the maximum tap value that still passes. */
1030         max = min + 1;
1031         while (max < 255) {
1032                 tegra_sdhci_set_tap(host, max);
1033                 if (mmc_send_tuning(host->mmc, opcode, NULL)) {
1034                         max--;
1035                         break;
1036                 }
1037                 max++;
1038         }
1039
1040         /* The TRM states the ideal tap value is at 75% in the passing range. */
1041         tegra_sdhci_set_tap(host, min + ((max - min) * 3 / 4));
1042
1043         return mmc_send_tuning(host->mmc, opcode, NULL);
1044 }
1045
1046 static int sdhci_tegra_start_signal_voltage_switch(struct mmc_host *mmc,
1047                                                    struct mmc_ios *ios)
1048 {
1049         struct sdhci_host *host = mmc_priv(mmc);
1050         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1051         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1052         int ret = 0;
1053
1054         if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
1055                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1056                 if (ret < 0)
1057                         return ret;
1058                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1059         } else if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180) {
1060                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1061                 if (ret < 0)
1062                         return ret;
1063                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1064         }
1065
1066         if (tegra_host->pad_calib_required)
1067                 tegra_sdhci_pad_autocalib(host);
1068
1069         return ret;
1070 }
1071
1072 static int tegra_sdhci_init_pinctrl_info(struct device *dev,
1073                                          struct sdhci_tegra *tegra_host)
1074 {
1075         tegra_host->pinctrl_sdmmc = devm_pinctrl_get(dev);
1076         if (IS_ERR(tegra_host->pinctrl_sdmmc)) {
1077                 dev_dbg(dev, "No pinctrl info, err: %ld\n",
1078                         PTR_ERR(tegra_host->pinctrl_sdmmc));
1079                 return -1;
1080         }
1081
1082         tegra_host->pinctrl_state_1v8_drv = pinctrl_lookup_state(
1083                                 tegra_host->pinctrl_sdmmc, "sdmmc-1v8-drv");
1084         if (IS_ERR(tegra_host->pinctrl_state_1v8_drv)) {
1085                 if (PTR_ERR(tegra_host->pinctrl_state_1v8_drv) == -ENODEV)
1086                         tegra_host->pinctrl_state_1v8_drv = NULL;
1087         }
1088
1089         tegra_host->pinctrl_state_3v3_drv = pinctrl_lookup_state(
1090                                 tegra_host->pinctrl_sdmmc, "sdmmc-3v3-drv");
1091         if (IS_ERR(tegra_host->pinctrl_state_3v3_drv)) {
1092                 if (PTR_ERR(tegra_host->pinctrl_state_3v3_drv) == -ENODEV)
1093                         tegra_host->pinctrl_state_3v3_drv = NULL;
1094         }
1095
1096         tegra_host->pinctrl_state_3v3 =
1097                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-3v3");
1098         if (IS_ERR(tegra_host->pinctrl_state_3v3)) {
1099                 dev_warn(dev, "Missing 3.3V pad state, err: %ld\n",
1100                          PTR_ERR(tegra_host->pinctrl_state_3v3));
1101                 return -1;
1102         }
1103
1104         tegra_host->pinctrl_state_1v8 =
1105                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-1v8");
1106         if (IS_ERR(tegra_host->pinctrl_state_1v8)) {
1107                 dev_warn(dev, "Missing 1.8V pad state, err: %ld\n",
1108                          PTR_ERR(tegra_host->pinctrl_state_1v8));
1109                 return -1;
1110         }
1111
1112         tegra_host->pad_control_available = true;
1113
1114         return 0;
1115 }
1116
1117 static void tegra_sdhci_voltage_switch(struct sdhci_host *host)
1118 {
1119         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1120         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1121         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1122
1123         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1124                 tegra_host->pad_calib_required = true;
1125 }
1126
1127 static void tegra_cqhci_writel(struct cqhci_host *cq_host, u32 val, int reg)
1128 {
1129         struct mmc_host *mmc = cq_host->mmc;
1130         u8 ctrl;
1131         ktime_t timeout;
1132         bool timed_out;
1133
1134         /*
1135          * During CQE resume/unhalt, CQHCI driver unhalts CQE prior to
1136          * cqhci_host_ops enable where SDHCI DMA and BLOCK_SIZE registers need
1137          * to be re-configured.
1138          * Tegra CQHCI/SDHCI prevents write access to block size register when
1139          * CQE is unhalted. So handling CQE resume sequence here to configure
1140          * SDHCI block registers prior to exiting CQE halt state.
1141          */
1142         if (reg == CQHCI_CTL && !(val & CQHCI_HALT) &&
1143             cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) {
1144                 sdhci_cqe_enable(mmc);
1145                 writel(val, cq_host->mmio + reg);
1146                 timeout = ktime_add_us(ktime_get(), 50);
1147                 while (1) {
1148                         timed_out = ktime_compare(ktime_get(), timeout) > 0;
1149                         ctrl = cqhci_readl(cq_host, CQHCI_CTL);
1150                         if (!(ctrl & CQHCI_HALT) || timed_out)
1151                                 break;
1152                 }
1153                 /*
1154                  * CQE usually resumes very quick, but incase if Tegra CQE
1155                  * doesn't resume retry unhalt.
1156                  */
1157                 if (timed_out)
1158                         writel(val, cq_host->mmio + reg);
1159         } else {
1160                 writel(val, cq_host->mmio + reg);
1161         }
1162 }
1163
1164 static void sdhci_tegra_update_dcmd_desc(struct mmc_host *mmc,
1165                                          struct mmc_request *mrq, u64 *data)
1166 {
1167         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(mmc_priv(mmc));
1168         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1169         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1170
1171         if (soc_data->nvquirks & NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING &&
1172             mrq->cmd->flags & MMC_RSP_R1B)
1173                 *data |= CQHCI_CMD_TIMING(1);
1174 }
1175
1176 static void sdhci_tegra_cqe_enable(struct mmc_host *mmc)
1177 {
1178         struct cqhci_host *cq_host = mmc->cqe_private;
1179         u32 val;
1180
1181         /*
1182          * Tegra CQHCI/SDMMC design prevents write access to sdhci block size
1183          * register when CQE is enabled and unhalted.
1184          * CQHCI driver enables CQE prior to activation, so disable CQE before
1185          * programming block size in sdhci controller and enable it back.
1186          */
1187         if (!cq_host->activated) {
1188                 val = cqhci_readl(cq_host, CQHCI_CFG);
1189                 if (val & CQHCI_ENABLE)
1190                         cqhci_writel(cq_host, (val & ~CQHCI_ENABLE),
1191                                      CQHCI_CFG);
1192                 sdhci_cqe_enable(mmc);
1193                 if (val & CQHCI_ENABLE)
1194                         cqhci_writel(cq_host, val, CQHCI_CFG);
1195         }
1196
1197         /*
1198          * CMD CRC errors are seen sometimes with some eMMC devices when status
1199          * command is sent during transfer of last data block which is the
1200          * default case as send status command block counter (CBC) is 1.
1201          * Recommended fix to set CBC to 0 allowing send status command only
1202          * when data lines are idle.
1203          */
1204         val = cqhci_readl(cq_host, CQHCI_SSC1);
1205         val &= ~CQHCI_SSC1_CBC_MASK;
1206         cqhci_writel(cq_host, val, CQHCI_SSC1);
1207 }
1208
1209 static void sdhci_tegra_dumpregs(struct mmc_host *mmc)
1210 {
1211         sdhci_dumpregs(mmc_priv(mmc));
1212 }
1213
1214 static u32 sdhci_tegra_cqhci_irq(struct sdhci_host *host, u32 intmask)
1215 {
1216         int cmd_error = 0;
1217         int data_error = 0;
1218
1219         if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
1220                 return intmask;
1221
1222         cqhci_irq(host->mmc, intmask, cmd_error, data_error);
1223
1224         return 0;
1225 }
1226
1227 static const struct cqhci_host_ops sdhci_tegra_cqhci_ops = {
1228         .write_l    = tegra_cqhci_writel,
1229         .enable = sdhci_tegra_cqe_enable,
1230         .disable = sdhci_cqe_disable,
1231         .dumpregs = sdhci_tegra_dumpregs,
1232         .update_dcmd_desc = sdhci_tegra_update_dcmd_desc,
1233 };
1234
1235 static const struct sdhci_ops tegra_sdhci_ops = {
1236         .read_w     = tegra_sdhci_readw,
1237         .write_l    = tegra_sdhci_writel,
1238         .set_clock  = tegra_sdhci_set_clock,
1239         .set_bus_width = sdhci_set_bus_width,
1240         .reset      = tegra_sdhci_reset,
1241         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1242         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1243         .voltage_switch = tegra_sdhci_voltage_switch,
1244         .get_max_clock = tegra_sdhci_get_max_clock,
1245 };
1246
1247 static const struct sdhci_pltfm_data sdhci_tegra20_pdata = {
1248         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1249                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1250                   SDHCI_QUIRK_NO_HISPD_BIT |
1251                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1252                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1253         .ops  = &tegra_sdhci_ops,
1254 };
1255
1256 static const struct sdhci_tegra_soc_data soc_data_tegra20 = {
1257         .pdata = &sdhci_tegra20_pdata,
1258         .nvquirks = NVQUIRK_FORCE_SDHCI_SPEC_200 |
1259                     NVQUIRK_ENABLE_BLOCK_GAP_DET,
1260 };
1261
1262 static const struct sdhci_pltfm_data sdhci_tegra30_pdata = {
1263         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1264                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1265                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1266                   SDHCI_QUIRK_NO_HISPD_BIT |
1267                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1268                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1269         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1270                    SDHCI_QUIRK2_BROKEN_HS200 |
1271                    /*
1272                     * Auto-CMD23 leads to "Got command interrupt 0x00010000 even
1273                     * though no command operation was in progress."
1274                     *
1275                     * The exact reason is unknown, as the same hardware seems
1276                     * to support Auto CMD23 on a downstream 3.1 kernel.
1277                     */
1278                    SDHCI_QUIRK2_ACMD23_BROKEN,
1279         .ops  = &tegra_sdhci_ops,
1280 };
1281
1282 static const struct sdhci_tegra_soc_data soc_data_tegra30 = {
1283         .pdata = &sdhci_tegra30_pdata,
1284         .nvquirks = NVQUIRK_ENABLE_SDHCI_SPEC_300 |
1285                     NVQUIRK_ENABLE_SDR50 |
1286                     NVQUIRK_ENABLE_SDR104 |
1287                     NVQUIRK_HAS_PADCALIB,
1288 };
1289
1290 static const struct sdhci_ops tegra114_sdhci_ops = {
1291         .read_w     = tegra_sdhci_readw,
1292         .write_w    = tegra_sdhci_writew,
1293         .write_l    = tegra_sdhci_writel,
1294         .set_clock  = tegra_sdhci_set_clock,
1295         .set_bus_width = sdhci_set_bus_width,
1296         .reset      = tegra_sdhci_reset,
1297         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1298         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1299         .voltage_switch = tegra_sdhci_voltage_switch,
1300         .get_max_clock = tegra_sdhci_get_max_clock,
1301 };
1302
1303 static const struct sdhci_pltfm_data sdhci_tegra114_pdata = {
1304         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1305                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1306                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1307                   SDHCI_QUIRK_NO_HISPD_BIT |
1308                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1309                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1310         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1311         .ops  = &tegra114_sdhci_ops,
1312 };
1313
1314 static const struct sdhci_tegra_soc_data soc_data_tegra114 = {
1315         .pdata = &sdhci_tegra114_pdata,
1316 };
1317
1318 static const struct sdhci_pltfm_data sdhci_tegra124_pdata = {
1319         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1320                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1321                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1322                   SDHCI_QUIRK_NO_HISPD_BIT |
1323                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1324                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1325         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1326                    /*
1327                     * The TRM states that the SD/MMC controller found on
1328                     * Tegra124 can address 34 bits (the maximum supported by
1329                     * the Tegra memory controller), but tests show that DMA
1330                     * to or from above 4 GiB doesn't work. This is possibly
1331                     * caused by missing programming, though it's not obvious
1332                     * what sequence is required. Mark 64-bit DMA broken for
1333                     * now to fix this for existing users (e.g. Nyan boards).
1334                     */
1335                    SDHCI_QUIRK2_BROKEN_64_BIT_DMA,
1336         .ops  = &tegra114_sdhci_ops,
1337 };
1338
1339 static const struct sdhci_tegra_soc_data soc_data_tegra124 = {
1340         .pdata = &sdhci_tegra124_pdata,
1341 };
1342
1343 static const struct sdhci_ops tegra210_sdhci_ops = {
1344         .read_w     = tegra_sdhci_readw,
1345         .write_w    = tegra210_sdhci_writew,
1346         .write_l    = tegra_sdhci_writel,
1347         .set_clock  = tegra_sdhci_set_clock,
1348         .set_bus_width = sdhci_set_bus_width,
1349         .reset      = tegra_sdhci_reset,
1350         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1351         .voltage_switch = tegra_sdhci_voltage_switch,
1352         .get_max_clock = tegra_sdhci_get_max_clock,
1353 };
1354
1355 static const struct sdhci_pltfm_data sdhci_tegra210_pdata = {
1356         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1357                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1358                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1359                   SDHCI_QUIRK_NO_HISPD_BIT |
1360                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1361                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1362         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1363         .ops  = &tegra210_sdhci_ops,
1364 };
1365
1366 static const struct sdhci_tegra_soc_data soc_data_tegra210 = {
1367         .pdata = &sdhci_tegra210_pdata,
1368         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1369                     NVQUIRK_HAS_PADCALIB |
1370                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1371                     NVQUIRK_ENABLE_SDR50 |
1372                     NVQUIRK_ENABLE_SDR104,
1373         .min_tap_delay = 106,
1374         .max_tap_delay = 185,
1375 };
1376
1377 static const struct sdhci_ops tegra186_sdhci_ops = {
1378         .read_w     = tegra_sdhci_readw,
1379         .write_l    = tegra_sdhci_writel,
1380         .set_clock  = tegra_sdhci_set_clock,
1381         .set_bus_width = sdhci_set_bus_width,
1382         .reset      = tegra_sdhci_reset,
1383         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1384         .voltage_switch = tegra_sdhci_voltage_switch,
1385         .get_max_clock = tegra_sdhci_get_max_clock,
1386         .irq = sdhci_tegra_cqhci_irq,
1387 };
1388
1389 static const struct sdhci_pltfm_data sdhci_tegra186_pdata = {
1390         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1391                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1392                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1393                   SDHCI_QUIRK_NO_HISPD_BIT |
1394                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1395                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1396         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1397                    /* SDHCI controllers on Tegra186 support 40-bit addressing.
1398                     * IOVA addresses are 48-bit wide on Tegra186.
1399                     * With 64-bit dma mask used for SDHCI, accesses can
1400                     * be broken. Disable 64-bit dma, which would fall back
1401                     * to 32-bit dma mask. Ideally 40-bit dma mask would work,
1402                     * But it is not supported as of now.
1403                     */
1404                    SDHCI_QUIRK2_BROKEN_64_BIT_DMA,
1405         .ops  = &tegra186_sdhci_ops,
1406 };
1407
1408 static const struct sdhci_tegra_soc_data soc_data_tegra186 = {
1409         .pdata = &sdhci_tegra186_pdata,
1410         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1411                     NVQUIRK_HAS_PADCALIB |
1412                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1413                     NVQUIRK_ENABLE_SDR50 |
1414                     NVQUIRK_ENABLE_SDR104 |
1415                     NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING,
1416         .min_tap_delay = 84,
1417         .max_tap_delay = 136,
1418 };
1419
1420 static const struct sdhci_tegra_soc_data soc_data_tegra194 = {
1421         .pdata = &sdhci_tegra186_pdata,
1422         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1423                     NVQUIRK_HAS_PADCALIB |
1424                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1425                     NVQUIRK_ENABLE_SDR50 |
1426                     NVQUIRK_ENABLE_SDR104,
1427         .min_tap_delay = 96,
1428         .max_tap_delay = 139,
1429 };
1430
1431 static const struct of_device_id sdhci_tegra_dt_match[] = {
1432         { .compatible = "nvidia,tegra194-sdhci", .data = &soc_data_tegra194 },
1433         { .compatible = "nvidia,tegra186-sdhci", .data = &soc_data_tegra186 },
1434         { .compatible = "nvidia,tegra210-sdhci", .data = &soc_data_tegra210 },
1435         { .compatible = "nvidia,tegra124-sdhci", .data = &soc_data_tegra124 },
1436         { .compatible = "nvidia,tegra114-sdhci", .data = &soc_data_tegra114 },
1437         { .compatible = "nvidia,tegra30-sdhci", .data = &soc_data_tegra30 },
1438         { .compatible = "nvidia,tegra20-sdhci", .data = &soc_data_tegra20 },
1439         {}
1440 };
1441 MODULE_DEVICE_TABLE(of, sdhci_tegra_dt_match);
1442
1443 static int sdhci_tegra_add_host(struct sdhci_host *host)
1444 {
1445         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1446         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1447         struct cqhci_host *cq_host;
1448         bool dma64;
1449         int ret;
1450
1451         if (!tegra_host->enable_hwcq)
1452                 return sdhci_add_host(host);
1453
1454         sdhci_enable_v4_mode(host);
1455
1456         ret = sdhci_setup_host(host);
1457         if (ret)
1458                 return ret;
1459
1460         host->mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
1461
1462         cq_host = devm_kzalloc(host->mmc->parent,
1463                                 sizeof(*cq_host), GFP_KERNEL);
1464         if (!cq_host) {
1465                 ret = -ENOMEM;
1466                 goto cleanup;
1467         }
1468
1469         cq_host->mmio = host->ioaddr + SDHCI_TEGRA_CQE_BASE_ADDR;
1470         cq_host->ops = &sdhci_tegra_cqhci_ops;
1471
1472         dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
1473         if (dma64)
1474                 cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
1475
1476         ret = cqhci_init(cq_host, host->mmc, dma64);
1477         if (ret)
1478                 goto cleanup;
1479
1480         ret = __sdhci_add_host(host);
1481         if (ret)
1482                 goto cleanup;
1483
1484         return 0;
1485
1486 cleanup:
1487         sdhci_cleanup_host(host);
1488         return ret;
1489 }
1490
1491 static int sdhci_tegra_probe(struct platform_device *pdev)
1492 {
1493         const struct of_device_id *match;
1494         const struct sdhci_tegra_soc_data *soc_data;
1495         struct sdhci_host *host;
1496         struct sdhci_pltfm_host *pltfm_host;
1497         struct sdhci_tegra *tegra_host;
1498         struct clk *clk;
1499         int rc;
1500
1501         match = of_match_device(sdhci_tegra_dt_match, &pdev->dev);
1502         if (!match)
1503                 return -EINVAL;
1504         soc_data = match->data;
1505
1506         host = sdhci_pltfm_init(pdev, soc_data->pdata, sizeof(*tegra_host));
1507         if (IS_ERR(host))
1508                 return PTR_ERR(host);
1509         pltfm_host = sdhci_priv(host);
1510
1511         tegra_host = sdhci_pltfm_priv(pltfm_host);
1512         tegra_host->ddr_signaling = false;
1513         tegra_host->pad_calib_required = false;
1514         tegra_host->pad_control_available = false;
1515         tegra_host->soc_data = soc_data;
1516
1517         if (soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL) {
1518                 rc = tegra_sdhci_init_pinctrl_info(&pdev->dev, tegra_host);
1519                 if (rc == 0)
1520                         host->mmc_host_ops.start_signal_voltage_switch =
1521                                 sdhci_tegra_start_signal_voltage_switch;
1522         }
1523
1524         /* Hook to periodically rerun pad calibration */
1525         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1526                 host->mmc_host_ops.request = tegra_sdhci_request;
1527
1528         host->mmc_host_ops.hs400_enhanced_strobe =
1529                         tegra_sdhci_hs400_enhanced_strobe;
1530
1531         if (!host->ops->platform_execute_tuning)
1532                 host->mmc_host_ops.execute_tuning =
1533                                 tegra_sdhci_execute_hw_tuning;
1534
1535         rc = mmc_of_parse(host->mmc);
1536         if (rc)
1537                 goto err_parse_dt;
1538
1539         if (tegra_host->soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
1540                 host->mmc->caps |= MMC_CAP_1_8V_DDR;
1541
1542         tegra_sdhci_parse_dt(host);
1543
1544         tegra_host->power_gpio = devm_gpiod_get_optional(&pdev->dev, "power",
1545                                                          GPIOD_OUT_HIGH);
1546         if (IS_ERR(tegra_host->power_gpio)) {
1547                 rc = PTR_ERR(tegra_host->power_gpio);
1548                 goto err_power_req;
1549         }
1550
1551         clk = devm_clk_get(mmc_dev(host->mmc), NULL);
1552         if (IS_ERR(clk)) {
1553                 dev_err(mmc_dev(host->mmc), "clk err\n");
1554                 rc = PTR_ERR(clk);
1555                 goto err_clk_get;
1556         }
1557         clk_prepare_enable(clk);
1558         pltfm_host->clk = clk;
1559
1560         tegra_host->rst = devm_reset_control_get_exclusive(&pdev->dev,
1561                                                            "sdhci");
1562         if (IS_ERR(tegra_host->rst)) {
1563                 rc = PTR_ERR(tegra_host->rst);
1564                 dev_err(&pdev->dev, "failed to get reset control: %d\n", rc);
1565                 goto err_rst_get;
1566         }
1567
1568         rc = reset_control_assert(tegra_host->rst);
1569         if (rc)
1570                 goto err_rst_get;
1571
1572         usleep_range(2000, 4000);
1573
1574         rc = reset_control_deassert(tegra_host->rst);
1575         if (rc)
1576                 goto err_rst_get;
1577
1578         usleep_range(2000, 4000);
1579
1580         rc = sdhci_tegra_add_host(host);
1581         if (rc)
1582                 goto err_add_host;
1583
1584         return 0;
1585
1586 err_add_host:
1587         reset_control_assert(tegra_host->rst);
1588 err_rst_get:
1589         clk_disable_unprepare(pltfm_host->clk);
1590 err_clk_get:
1591 err_power_req:
1592 err_parse_dt:
1593         sdhci_pltfm_free(pdev);
1594         return rc;
1595 }
1596
1597 static int sdhci_tegra_remove(struct platform_device *pdev)
1598 {
1599         struct sdhci_host *host = platform_get_drvdata(pdev);
1600         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1601         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1602
1603         sdhci_remove_host(host, 0);
1604
1605         reset_control_assert(tegra_host->rst);
1606         usleep_range(2000, 4000);
1607         clk_disable_unprepare(pltfm_host->clk);
1608
1609         sdhci_pltfm_free(pdev);
1610
1611         return 0;
1612 }
1613
1614 #ifdef CONFIG_PM_SLEEP
1615 static int __maybe_unused sdhci_tegra_suspend(struct device *dev)
1616 {
1617         struct sdhci_host *host = dev_get_drvdata(dev);
1618         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1619         int ret;
1620
1621         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1622                 ret = cqhci_suspend(host->mmc);
1623                 if (ret)
1624                         return ret;
1625         }
1626
1627         ret = sdhci_suspend_host(host);
1628         if (ret) {
1629                 cqhci_resume(host->mmc);
1630                 return ret;
1631         }
1632
1633         clk_disable_unprepare(pltfm_host->clk);
1634         return 0;
1635 }
1636
1637 static int __maybe_unused sdhci_tegra_resume(struct device *dev)
1638 {
1639         struct sdhci_host *host = dev_get_drvdata(dev);
1640         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1641         int ret;
1642
1643         ret = clk_prepare_enable(pltfm_host->clk);
1644         if (ret)
1645                 return ret;
1646
1647         ret = sdhci_resume_host(host);
1648         if (ret)
1649                 goto disable_clk;
1650
1651         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1652                 ret = cqhci_resume(host->mmc);
1653                 if (ret)
1654                         goto suspend_host;
1655         }
1656
1657         return 0;
1658
1659 suspend_host:
1660         sdhci_suspend_host(host);
1661 disable_clk:
1662         clk_disable_unprepare(pltfm_host->clk);
1663         return ret;
1664 }
1665 #endif
1666
1667 static SIMPLE_DEV_PM_OPS(sdhci_tegra_dev_pm_ops, sdhci_tegra_suspend,
1668                          sdhci_tegra_resume);
1669
1670 static struct platform_driver sdhci_tegra_driver = {
1671         .driver         = {
1672                 .name   = "sdhci-tegra",
1673                 .of_match_table = sdhci_tegra_dt_match,
1674                 .pm     = &sdhci_tegra_dev_pm_ops,
1675         },
1676         .probe          = sdhci_tegra_probe,
1677         .remove         = sdhci_tegra_remove,
1678 };
1679
1680 module_platform_driver(sdhci_tegra_driver);
1681
1682 MODULE_DESCRIPTION("SDHCI driver for Tegra");
1683 MODULE_AUTHOR("Google, Inc.");
1684 MODULE_LICENSE("GPL v2");