drivers/mmc/host/sdhci-tegra.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2010 Google, Inc.
   4  */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/clk.h>
   8 #include <linux/delay.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/err.h>
  11 #include <linux/gpio/consumer.h>
  12 #include <linux/init.h>
  13 #include <linux/io.h>
  14 #include <linux/iommu.h>
  15 #include <linux/iopoll.h>
  16 #include <linux/ktime.h>
  17 #include <linux/mmc/card.h>
  18 #include <linux/mmc/host.h>
  19 #include <linux/mmc/mmc.h>
  20 #include <linux/mmc/slot-gpio.h>
  21 #include <linux/module.h>
  22 #include <linux/of_device.h>
  23 #include <linux/of.h>
  24 #include <linux/pinctrl/consumer.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm_opp.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/regulator/consumer.h>
  29 #include <linux/reset.h>
  30
  31 #include <soc/tegra/common.h>
  32
  33 #include "sdhci-cqhci.h"
  34 #include "sdhci-pltfm.h"
  35 #include "cqhci.h"
  36
  37 /* Tegra SDHOST controller vendor register definitions */
  38 #define SDHCI_TEGRA_VENDOR_CLOCK_CTRL                   0x100
  39 #define SDHCI_CLOCK_CTRL_TAP_MASK                       0x00ff0000
  40 #define SDHCI_CLOCK_CTRL_TAP_SHIFT                      16
  41 #define SDHCI_CLOCK_CTRL_TRIM_MASK                      0x1f000000
  42 #define SDHCI_CLOCK_CTRL_TRIM_SHIFT                     24
  43 #define SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE          BIT(5)
  44 #define SDHCI_CLOCK_CTRL_PADPIPE_CLKEN_OVERRIDE         BIT(3)
  45 #define SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE        BIT(2)
  46
  47 #define SDHCI_TEGRA_VENDOR_SYS_SW_CTRL                  0x104
  48 #define SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE         BIT(31)
  49
  50 #define SDHCI_TEGRA_VENDOR_CAP_OVERRIDES                0x10c
  51 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK         0x00003f00
  52 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT        8
  53
  54 #define SDHCI_TEGRA_VENDOR_MISC_CTRL                    0x120
  55 #define SDHCI_MISC_CTRL_ERASE_TIMEOUT_LIMIT             BIT(0)
  56 #define SDHCI_MISC_CTRL_ENABLE_SDR104                   0x8
  57 #define SDHCI_MISC_CTRL_ENABLE_SDR50                    0x10
  58 #define SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300           0x20
  59 #define SDHCI_MISC_CTRL_ENABLE_DDR50                    0x200
  60
  61 #define SDHCI_TEGRA_VENDOR_DLLCAL_CFG                   0x1b0
  62 #define SDHCI_TEGRA_DLLCAL_CALIBRATE                    BIT(31)
  63
  64 #define SDHCI_TEGRA_VENDOR_DLLCAL_STA                   0x1bc
  65 #define SDHCI_TEGRA_DLLCAL_STA_ACTIVE                   BIT(31)
  66
  67 #define SDHCI_VNDR_TUN_CTRL0_0                          0x1c0
  68 #define SDHCI_VNDR_TUN_CTRL0_TUN_HW_TAP                 0x20000
  69 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK         0x03fc0000
  70 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT        18
  71 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK                 0x00001fc0
  72 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT                6
  73 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK              0x000e000
  74 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT             13
  75 #define TRIES_128                                       2
  76 #define TRIES_256                                       4
  77 #define SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK          0x7
  78
  79 #define SDHCI_TEGRA_VNDR_TUN_CTRL1_0                    0x1c4
  80 #define SDHCI_TEGRA_VNDR_TUN_STATUS0                    0x1C8
  81 #define SDHCI_TEGRA_VNDR_TUN_STATUS1                    0x1CC
  82 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK           0xFF
  83 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT      0x8
  84 #define TUNING_WORD_BIT_SIZE                            32
  85
  86 #define SDHCI_TEGRA_AUTO_CAL_CONFIG                     0x1e4
  87 #define SDHCI_AUTO_CAL_START                            BIT(31)
  88 #define SDHCI_AUTO_CAL_ENABLE                           BIT(29)
  89 #define SDHCI_AUTO_CAL_PDPU_OFFSET_MASK                 0x0000ffff
  90
  91 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL                  0x1e0
  92 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK    0x0000000f
  93 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL     0x7
  94 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD   BIT(31)
  95 #define SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK          0x07FFF000
  96
  97 #define SDHCI_TEGRA_AUTO_CAL_STATUS                     0x1ec
  98 #define SDHCI_TEGRA_AUTO_CAL_ACTIVE                     BIT(31)
  99
 100 #define SDHCI_TEGRA_CIF2AXI_CTRL_0                      0x1fc
 101
 102 #define NVQUIRK_FORCE_SDHCI_SPEC_200                    BIT(0)
 103 #define NVQUIRK_ENABLE_BLOCK_GAP_DET                    BIT(1)
 104 #define NVQUIRK_ENABLE_SDHCI_SPEC_300                   BIT(2)
 105 #define NVQUIRK_ENABLE_SDR50                            BIT(3)
 106 #define NVQUIRK_ENABLE_SDR104                           BIT(4)
 107 #define NVQUIRK_ENABLE_DDR50                            BIT(5)
 108 /*
 109  * HAS_PADCALIB NVQUIRK is for SoC's supporting auto calibration of pads
 110  * drive strength.
 111  */
 112 #define NVQUIRK_HAS_PADCALIB                            BIT(6)
 113 /*
 114  * NEEDS_PAD_CONTROL NVQUIRK is for SoC's having separate 3V3 and 1V8 pads.
 115  * 3V3/1V8 pad selection happens through pinctrl state selection depending
 116  * on the signaling mode.
 117  */
 118 #define NVQUIRK_NEEDS_PAD_CONTROL                       BIT(7)
 119 #define NVQUIRK_DIS_CARD_CLK_CONFIG_TAP                 BIT(8)
 120 #define NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING               BIT(9)
 121
 122 /*
 123  * NVQUIRK_HAS_TMCLK is for SoC's having separate timeout clock for Tegra
 124  * SDMMC hardware data timeout.
 125  */
 126 #define NVQUIRK_HAS_TMCLK                               BIT(10)
 127
 128 #define NVQUIRK_HAS_ANDROID_GPT_SECTOR                  BIT(11)
 129 #define NVQUIRK_PROGRAM_STREAMID                        BIT(12)
 130
 131 /* SDMMC CQE Base Address for Tegra Host Ver 4.1 and Higher */
 132 #define SDHCI_TEGRA_CQE_BASE_ADDR                       0xF000
 133
 134 #define SDHCI_TEGRA_CQE_TRNS_MODE       (SDHCI_TRNS_MULTI | \
 135                                          SDHCI_TRNS_BLK_CNT_EN | \
 136                                          SDHCI_TRNS_DMA)
 137
 138 struct sdhci_tegra_soc_data {
 139         const struct sdhci_pltfm_data *pdata;
 140         u64 dma_mask;
 141         u32 nvquirks;
 142         u8 min_tap_delay;
 143         u8 max_tap_delay;
 144 };
 145
 146 /* Magic pull up and pull down pad calibration offsets */
 147 struct sdhci_tegra_autocal_offsets {
 148         u32 pull_up_3v3;
 149         u32 pull_down_3v3;
 150         u32 pull_up_3v3_timeout;
 151         u32 pull_down_3v3_timeout;
 152         u32 pull_up_1v8;
 153         u32 pull_down_1v8;
 154         u32 pull_up_1v8_timeout;
 155         u32 pull_down_1v8_timeout;
 156         u32 pull_up_sdr104;
 157         u32 pull_down_sdr104;
 158         u32 pull_up_hs400;
 159         u32 pull_down_hs400;
 160 };
 161
 162 struct sdhci_tegra {
 163         const struct sdhci_tegra_soc_data *soc_data;
 164         struct gpio_desc *power_gpio;
 165         struct clk *tmclk;
 166         bool ddr_signaling;
 167         bool pad_calib_required;
 168         bool pad_control_available;
 169
 170         struct reset_control *rst;
 171         struct pinctrl *pinctrl_sdmmc;
 172         struct pinctrl_state *pinctrl_state_3v3;
 173         struct pinctrl_state *pinctrl_state_1v8;
 174         struct pinctrl_state *pinctrl_state_3v3_drv;
 175         struct pinctrl_state *pinctrl_state_1v8_drv;
 176
 177         struct sdhci_tegra_autocal_offsets autocal_offsets;
 178         ktime_t last_calib;
 179
 180         u32 default_tap;
 181         u32 default_trim;
 182         u32 dqs_trim;
 183         bool enable_hwcq;
 184         unsigned long curr_clk_rate;
 185         u8 tuned_tap_delay;
 186         u32 stream_id;
 187 };
 188
 189 static u16 tegra_sdhci_readw(struct sdhci_host *host, int reg)
 190 {
 191         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 192         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 193         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 194
 195         if (unlikely((soc_data->nvquirks & NVQUIRK_FORCE_SDHCI_SPEC_200) &&
 196                         (reg == SDHCI_HOST_VERSION))) {
 197                 /* Erratum: Version register is invalid in HW. */
 198                 return SDHCI_SPEC_200;
 199         }
 200
 201         return readw(host->ioaddr + reg);
 202 }
 203
 204 static void tegra_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 205 {
 206         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 207
 208         switch (reg) {
 209         case SDHCI_TRANSFER_MODE:
 210                 /*
 211                  * Postpone this write, we must do it together with a
 212                  * command write that is down below.
 213                  */
 214                 pltfm_host->xfer_mode_shadow = val;
 215                 return;
 216         case SDHCI_COMMAND:
 217                 writel((val << 16) | pltfm_host->xfer_mode_shadow,
 218                         host->ioaddr + SDHCI_TRANSFER_MODE);
 219                 return;
 220         }
 221
 222         writew(val, host->ioaddr + reg);
 223 }
 224
 225 static void tegra_sdhci_writel(struct sdhci_host *host, u32 val, int reg)
 226 {
 227         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 228         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 229         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 230
 231         /* Seems like we're getting spurious timeout and crc errors, so
 232          * disable signalling of them. In case of real errors software
 233          * timers should take care of eventually detecting them.
 234          */
 235         if (unlikely(reg == SDHCI_SIGNAL_ENABLE))
 236                 val &= ~(SDHCI_INT_TIMEOUT|SDHCI_INT_CRC);
 237
 238         writel(val, host->ioaddr + reg);
 239
 240         if (unlikely((soc_data->nvquirks & NVQUIRK_ENABLE_BLOCK_GAP_DET) &&
 241                         (reg == SDHCI_INT_ENABLE))) {
 242                 /* Erratum: Must enable block gap interrupt detection */
 243                 u8 gap_ctrl = readb(host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 244                 if (val & SDHCI_INT_CARD_INT)
 245                         gap_ctrl |= 0x8;
 246                 else
 247                         gap_ctrl &= ~0x8;
 248                 writeb(gap_ctrl, host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 249         }
 250 }
 251
 252 static bool tegra_sdhci_configure_card_clk(struct sdhci_host *host, bool enable)
 253 {
 254         bool status;
 255         u32 reg;
 256
 257         reg = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
 258         status = !!(reg & SDHCI_CLOCK_CARD_EN);
 259
 260         if (status == enable)
 261                 return status;
 262
 263         if (enable)
 264                 reg |= SDHCI_CLOCK_CARD_EN;
 265         else
 266                 reg &= ~SDHCI_CLOCK_CARD_EN;
 267
 268         sdhci_writew(host, reg, SDHCI_CLOCK_CONTROL);
 269
 270         return status;
 271 }
 272
 273 static void tegra210_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 274 {
 275         bool is_tuning_cmd = 0;
 276         bool clk_enabled;
 277
 278         if (reg == SDHCI_COMMAND)
 279                 is_tuning_cmd = mmc_op_tuning(SDHCI_GET_CMD(val));
 280
 281         if (is_tuning_cmd)
 282                 clk_enabled = tegra_sdhci_configure_card_clk(host, 0);
 283
 284         writew(val, host->ioaddr + reg);
 285
 286         if (is_tuning_cmd) {
 287                 udelay(1);
 288                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 289                 tegra_sdhci_configure_card_clk(host, clk_enabled);
 290         }
 291 }
 292
 293 static unsigned int tegra_sdhci_get_ro(struct sdhci_host *host)
 294 {
 295         /*
 296          * Write-enable shall be assumed if GPIO is missing in a board's
 297          * device-tree because SDHCI's WRITE_PROTECT bit doesn't work on
 298          * Tegra.
 299          */
 300         return mmc_gpio_get_ro(host->mmc);
 301 }
 302
 303 static bool tegra_sdhci_is_pad_and_regulator_valid(struct sdhci_host *host)
 304 {
 305         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 306         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 307         int has_1v8, has_3v3;
 308
 309         /*
 310          * The SoCs which have NVQUIRK_NEEDS_PAD_CONTROL require software pad
 311          * voltage configuration in order to perform voltage switching. This
 312          * means that valid pinctrl info is required on SDHCI instances capable
 313          * of performing voltage switching. Whether or not an SDHCI instance is
 314          * capable of voltage switching is determined based on the regulator.
 315          */
 316
 317         if (!(tegra_host->soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL))
 318                 return true;
 319
 320         if (IS_ERR(host->mmc->supply.vqmmc))
 321                 return false;
 322
 323         has_1v8 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 324                                                  1700000, 1950000);
 325
 326         has_3v3 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 327                                                  2700000, 3600000);
 328
 329         if (has_1v8 == 1 && has_3v3 == 1)
 330                 return tegra_host->pad_control_available;
 331
 332         /* Fixed voltage, no pad control required. */
 333         return true;
 334 }
 335
 336 static void tegra_sdhci_set_tap(struct sdhci_host *host, unsigned int tap)
 337 {
 338         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 339         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 340         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 341         bool card_clk_enabled = false;
 342         u32 reg;
 343
 344         /*
 345          * Touching the tap values is a bit tricky on some SoC generations.
 346          * The quirk enables a workaround for a glitch that sometimes occurs if
 347          * the tap values are changed.
 348          */
 349
 350         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP)
 351                 card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 352
 353         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 354         reg &= ~SDHCI_CLOCK_CTRL_TAP_MASK;
 355         reg |= tap << SDHCI_CLOCK_CTRL_TAP_SHIFT;
 356         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 357
 358         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP &&
 359             card_clk_enabled) {
 360                 udelay(1);
 361                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 362                 tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 363         }
 364 }
 365
 366 static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask)
 367 {
 368         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 369         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 370         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 371         u32 misc_ctrl, clk_ctrl, pad_ctrl;
 372
 373         sdhci_and_cqhci_reset(host, mask);
 374
 375         if (!(mask & SDHCI_RESET_ALL))
 376                 return;
 377
 378         tegra_sdhci_set_tap(host, tegra_host->default_tap);
 379
 380         misc_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 381         clk_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 382
 383         misc_ctrl &= ~(SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300 |
 384                        SDHCI_MISC_CTRL_ENABLE_SDR50 |
 385                        SDHCI_MISC_CTRL_ENABLE_DDR50 |
 386                        SDHCI_MISC_CTRL_ENABLE_SDR104);
 387
 388         clk_ctrl &= ~(SDHCI_CLOCK_CTRL_TRIM_MASK |
 389                       SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE);
 390
 391         if (tegra_sdhci_is_pad_and_regulator_valid(host)) {
 392                 /* Erratum: Enable SDHCI spec v3.00 support */
 393                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDHCI_SPEC_300)
 394                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300;
 395                 /* Advertise UHS modes as supported by host */
 396                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR50)
 397                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR50;
 398                 if (soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
 399                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_DDR50;
 400                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR104)
 401                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR104;
 402                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR50)
 403                         clk_ctrl |= SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE;
 404         }
 405
 406         clk_ctrl |= tegra_host->default_trim << SDHCI_CLOCK_CTRL_TRIM_SHIFT;
 407
 408         sdhci_writel(host, misc_ctrl, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 409         sdhci_writel(host, clk_ctrl, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 410
 411         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB) {
 412                 pad_ctrl = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 413                 pad_ctrl &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK;
 414                 pad_ctrl |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL;
 415                 sdhci_writel(host, pad_ctrl, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 416
 417                 tegra_host->pad_calib_required = true;
 418         }
 419
 420         tegra_host->ddr_signaling = false;
 421 }
 422
 423 static void tegra_sdhci_configure_cal_pad(struct sdhci_host *host, bool enable)
 424 {
 425         u32 val;
 426
 427         /*
 428          * Enable or disable the additional I/O pad used by the drive strength
 429          * calibration process.
 430          */
 431         val = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 432
 433         if (enable)
 434                 val |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 435         else
 436                 val &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 437
 438         sdhci_writel(host, val, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 439
 440         if (enable)
 441                 usleep_range(1, 2);
 442 }
 443
 444 static void tegra_sdhci_set_pad_autocal_offset(struct sdhci_host *host,
 445                                                u16 pdpu)
 446 {
 447         u32 reg;
 448
 449         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 450         reg &= ~SDHCI_AUTO_CAL_PDPU_OFFSET_MASK;
 451         reg |= pdpu;
 452         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 453 }
 454
 455 static int tegra_sdhci_set_padctrl(struct sdhci_host *host, int voltage,
 456                                    bool state_drvupdn)
 457 {
 458         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 459         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 460         struct sdhci_tegra_autocal_offsets *offsets =
 461                                                 &tegra_host->autocal_offsets;
 462         struct pinctrl_state *pinctrl_drvupdn = NULL;
 463         int ret = 0;
 464         u8 drvup = 0, drvdn = 0;
 465         u32 reg;
 466
 467         if (!state_drvupdn) {
 468                 /* PADS Drive Strength */
 469                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 470                         if (tegra_host->pinctrl_state_1v8_drv) {
 471                                 pinctrl_drvupdn =
 472                                         tegra_host->pinctrl_state_1v8_drv;
 473                         } else {
 474                                 drvup = offsets->pull_up_1v8_timeout;
 475                                 drvdn = offsets->pull_down_1v8_timeout;
 476                         }
 477                 } else {
 478                         if (tegra_host->pinctrl_state_3v3_drv) {
 479                                 pinctrl_drvupdn =
 480                                         tegra_host->pinctrl_state_3v3_drv;
 481                         } else {
 482                                 drvup = offsets->pull_up_3v3_timeout;
 483                                 drvdn = offsets->pull_down_3v3_timeout;
 484                         }
 485                 }
 486
 487                 if (pinctrl_drvupdn != NULL) {
 488                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 489                                                         pinctrl_drvupdn);
 490                         if (ret < 0)
 491                                 dev_err(mmc_dev(host->mmc),
 492                                         "failed pads drvupdn, ret: %d\n", ret);
 493                 } else if ((drvup) || (drvdn)) {
 494                         reg = sdhci_readl(host,
 495                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 496                         reg &= ~SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK;
 497                         reg |= (drvup << 20) | (drvdn << 12);
 498                         sdhci_writel(host, reg,
 499                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 500                 }
 501
 502         } else {
 503                 /* Dual Voltage PADS Voltage selection */
 504                 if (!tegra_host->pad_control_available)
 505                         return 0;
 506
 507                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 508                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 509                                                 tegra_host->pinctrl_state_1v8);
 510                         if (ret < 0)
 511                                 dev_err(mmc_dev(host->mmc),
 512                                         "setting 1.8V failed, ret: %d\n", ret);
 513                 } else {
 514                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 515                                                 tegra_host->pinctrl_state_3v3);
 516                         if (ret < 0)
 517                                 dev_err(mmc_dev(host->mmc),
 518                                         "setting 3.3V failed, ret: %d\n", ret);
 519                 }
 520         }
 521
 522         return ret;
 523 }
 524
 525 static void tegra_sdhci_pad_autocalib(struct sdhci_host *host)
 526 {
 527         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 528         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 529         struct sdhci_tegra_autocal_offsets offsets =
 530                         tegra_host->autocal_offsets;
 531         struct mmc_ios *ios = &host->mmc->ios;
 532         bool card_clk_enabled;
 533         u16 pdpu;
 534         u32 reg;
 535         int ret;
 536
 537         switch (ios->timing) {
 538         case MMC_TIMING_UHS_SDR104:
 539                 pdpu = offsets.pull_down_sdr104 << 8 | offsets.pull_up_sdr104;
 540                 break;
 541         case MMC_TIMING_MMC_HS400:
 542                 pdpu = offsets.pull_down_hs400 << 8 | offsets.pull_up_hs400;
 543                 break;
 544         default:
 545                 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
 546                         pdpu = offsets.pull_down_1v8 << 8 | offsets.pull_up_1v8;
 547                 else
 548                         pdpu = offsets.pull_down_3v3 << 8 | offsets.pull_up_3v3;
 549         }
 550
 551         /* Set initial offset before auto-calibration */
 552         tegra_sdhci_set_pad_autocal_offset(host, pdpu);
 553
 554         card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 555
 556         tegra_sdhci_configure_cal_pad(host, true);
 557
 558         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 559         reg |= SDHCI_AUTO_CAL_ENABLE | SDHCI_AUTO_CAL_START;
 560         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 561
 562         usleep_range(1, 2);
 563         /* 10 ms timeout */
 564         ret = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_AUTO_CAL_STATUS,
 565                                  reg, !(reg & SDHCI_TEGRA_AUTO_CAL_ACTIVE),
 566                                  1000, 10000);
 567
 568         tegra_sdhci_configure_cal_pad(host, false);
 569
 570         tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 571
 572         if (ret) {
 573                 dev_err(mmc_dev(host->mmc), "Pad autocal timed out\n");
 574
 575                 /* Disable automatic cal and use fixed Drive Strengths */
 576                 reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 577                 reg &= ~SDHCI_AUTO_CAL_ENABLE;
 578                 sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 579
 580                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, false);
 581                 if (ret < 0)
 582                         dev_err(mmc_dev(host->mmc),
 583                                 "Setting drive strengths failed: %d\n", ret);
 584         }
 585 }
 586
 587 static void tegra_sdhci_parse_pad_autocal_dt(struct sdhci_host *host)
 588 {
 589         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 590         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 591         struct sdhci_tegra_autocal_offsets *autocal =
 592                         &tegra_host->autocal_offsets;
 593         int err;
 594
 595         err = device_property_read_u32(mmc_dev(host->mmc),
 596                         "nvidia,pad-autocal-pull-up-offset-3v3",
 597                         &autocal->pull_up_3v3);
 598         if (err)
 599                 autocal->pull_up_3v3 = 0;
 600
 601         err = device_property_read_u32(mmc_dev(host->mmc),
 602                         "nvidia,pad-autocal-pull-down-offset-3v3",
 603                         &autocal->pull_down_3v3);
 604         if (err)
 605                 autocal->pull_down_3v3 = 0;
 606
 607         err = device_property_read_u32(mmc_dev(host->mmc),
 608                         "nvidia,pad-autocal-pull-up-offset-1v8",
 609                         &autocal->pull_up_1v8);
 610         if (err)
 611                 autocal->pull_up_1v8 = 0;
 612
 613         err = device_property_read_u32(mmc_dev(host->mmc),
 614                         "nvidia,pad-autocal-pull-down-offset-1v8",
 615                         &autocal->pull_down_1v8);
 616         if (err)
 617                 autocal->pull_down_1v8 = 0;
 618
 619         err = device_property_read_u32(mmc_dev(host->mmc),
 620                         "nvidia,pad-autocal-pull-up-offset-sdr104",
 621                         &autocal->pull_up_sdr104);
 622         if (err)
 623                 autocal->pull_up_sdr104 = autocal->pull_up_1v8;
 624
 625         err = device_property_read_u32(mmc_dev(host->mmc),
 626                         "nvidia,pad-autocal-pull-down-offset-sdr104",
 627                         &autocal->pull_down_sdr104);
 628         if (err)
 629                 autocal->pull_down_sdr104 = autocal->pull_down_1v8;
 630
 631         err = device_property_read_u32(mmc_dev(host->mmc),
 632                         "nvidia,pad-autocal-pull-up-offset-hs400",
 633                         &autocal->pull_up_hs400);
 634         if (err)
 635                 autocal->pull_up_hs400 = autocal->pull_up_1v8;
 636
 637         err = device_property_read_u32(mmc_dev(host->mmc),
 638                         "nvidia,pad-autocal-pull-down-offset-hs400",
 639                         &autocal->pull_down_hs400);
 640         if (err)
 641                 autocal->pull_down_hs400 = autocal->pull_down_1v8;
 642
 643         /*
 644          * Different fail-safe drive strength values based on the signaling
 645          * voltage are applicable for SoCs supporting 3V3 and 1V8 pad controls.
 646          * So, avoid reading below device tree properties for SoCs that don't
 647          * have NVQUIRK_NEEDS_PAD_CONTROL.
 648          */
 649         if (!(tegra_host->soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL))
 650                 return;
 651
 652         err = device_property_read_u32(mmc_dev(host->mmc),
 653                         "nvidia,pad-autocal-pull-up-offset-3v3-timeout",
 654                         &autocal->pull_up_3v3_timeout);
 655         if (err) {
 656                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 657                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 658                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 659                                 mmc_hostname(host->mmc));
 660                 autocal->pull_up_3v3_timeout = 0;
 661         }
 662
 663         err = device_property_read_u32(mmc_dev(host->mmc),
 664                         "nvidia,pad-autocal-pull-down-offset-3v3-timeout",
 665                         &autocal->pull_down_3v3_timeout);
 666         if (err) {
 667                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 668                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 669                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 670                                 mmc_hostname(host->mmc));
 671                 autocal->pull_down_3v3_timeout = 0;
 672         }
 673
 674         err = device_property_read_u32(mmc_dev(host->mmc),
 675                         "nvidia,pad-autocal-pull-up-offset-1v8-timeout",
 676                         &autocal->pull_up_1v8_timeout);
 677         if (err) {
 678                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 679                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 680                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 681                                 mmc_hostname(host->mmc));
 682                 autocal->pull_up_1v8_timeout = 0;
 683         }
 684
 685         err = device_property_read_u32(mmc_dev(host->mmc),
 686                         "nvidia,pad-autocal-pull-down-offset-1v8-timeout",
 687                         &autocal->pull_down_1v8_timeout);
 688         if (err) {
 689                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 690                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 691                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 692                                 mmc_hostname(host->mmc));
 693                 autocal->pull_down_1v8_timeout = 0;
 694         }
 695 }
 696
 697 static void tegra_sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 698 {
 699         struct sdhci_host *host = mmc_priv(mmc);
 700         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 701         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 702         ktime_t since_calib = ktime_sub(ktime_get(), tegra_host->last_calib);
 703
 704         /* 100 ms calibration interval is specified in the TRM */
 705         if (ktime_to_ms(since_calib) > 100) {
 706                 tegra_sdhci_pad_autocalib(host);
 707                 tegra_host->last_calib = ktime_get();
 708         }
 709
 710         sdhci_request(mmc, mrq);
 711 }
 712
 713 static void tegra_sdhci_parse_tap_and_trim(struct sdhci_host *host)
 714 {
 715         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 716         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 717         int err;
 718
 719         err = device_property_read_u32(mmc_dev(host->mmc), "nvidia,default-tap",
 720                                        &tegra_host->default_tap);
 721         if (err)
 722                 tegra_host->default_tap = 0;
 723
 724         err = device_property_read_u32(mmc_dev(host->mmc), "nvidia,default-trim",
 725                                        &tegra_host->default_trim);
 726         if (err)
 727                 tegra_host->default_trim = 0;
 728
 729         err = device_property_read_u32(mmc_dev(host->mmc), "nvidia,dqs-trim",
 730                                        &tegra_host->dqs_trim);
 731         if (err)
 732                 tegra_host->dqs_trim = 0x11;
 733 }
 734
 735 static void tegra_sdhci_parse_dt(struct sdhci_host *host)
 736 {
 737         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 738         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 739
 740         if (device_property_read_bool(mmc_dev(host->mmc), "supports-cqe"))
 741                 tegra_host->enable_hwcq = true;
 742         else
 743                 tegra_host->enable_hwcq = false;
 744
 745         tegra_sdhci_parse_pad_autocal_dt(host);
 746         tegra_sdhci_parse_tap_and_trim(host);
 747 }
 748
 749 static void tegra_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
 750 {
 751         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 752         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 753         struct device *dev = mmc_dev(host->mmc);
 754         unsigned long host_clk;
 755         int err;
 756
 757         if (!clock)
 758                 return sdhci_set_clock(host, clock);
 759
 760         /*
 761          * In DDR50/52 modes the Tegra SDHCI controllers require the SDHCI
 762          * divider to be configured to divided the host clock by two. The SDHCI
 763          * clock divider is calculated as part of sdhci_set_clock() by
 764          * sdhci_calc_clk(). The divider is calculated from host->max_clk and
 765          * the requested clock rate.
 766          *
 767          * By setting the host->max_clk to clock * 2 the divider calculation
 768          * will always result in the correct value for DDR50/52 modes,
 769          * regardless of clock rate rounding, which may happen if the value
 770          * from clk_get_rate() is used.
 771          */
 772         host_clk = tegra_host->ddr_signaling ? clock * 2 : clock;
 773
 774         err = dev_pm_opp_set_rate(dev, host_clk);
 775         if (err)
 776                 dev_err(dev, "failed to set clk rate to %luHz: %d\n",
 777                         host_clk, err);
 778
 779         tegra_host->curr_clk_rate = clk_get_rate(pltfm_host->clk);
 780         if (tegra_host->ddr_signaling)
 781                 host->max_clk = host_clk;
 782         else
 783                 host->max_clk = clk_get_rate(pltfm_host->clk);
 784
 785         sdhci_set_clock(host, clock);
 786
 787         if (tegra_host->pad_calib_required) {
 788                 tegra_sdhci_pad_autocalib(host);
 789                 tegra_host->pad_calib_required = false;
 790         }
 791 }
 792
 793 static void tegra_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
 794                                               struct mmc_ios *ios)
 795 {
 796         struct sdhci_host *host = mmc_priv(mmc);
 797         u32 val;
 798
 799         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 800
 801         if (ios->enhanced_strobe) {
 802                 val |= SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 803                 /*
 804                  * When CMD13 is sent from mmc_select_hs400es() after
 805                  * switching to HS400ES mode, the bus is operating at
 806                  * either MMC_HIGH_26_MAX_DTR or MMC_HIGH_52_MAX_DTR.
 807                  * To meet Tegra SDHCI requirement at HS400ES mode, force SDHCI
 808                  * interface clock to MMC_HS200_MAX_DTR (200 MHz) so that host
 809                  * controller CAR clock and the interface clock are rate matched.
 810                  */
 811                 tegra_sdhci_set_clock(host, MMC_HS200_MAX_DTR);
 812         } else {
 813                 val &= ~SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 814         }
 815
 816         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 817 }
 818
 819 static unsigned int tegra_sdhci_get_max_clock(struct sdhci_host *host)
 820 {
 821         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 822
 823         return clk_round_rate(pltfm_host->clk, UINT_MAX);
 824 }
 825
 826 static void tegra_sdhci_set_dqs_trim(struct sdhci_host *host, u8 trim)
 827 {
 828         u32 val;
 829
 830         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 831         val &= ~SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK;
 832         val |= trim << SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT;
 833         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 834 }
 835
 836 static void tegra_sdhci_hs400_dll_cal(struct sdhci_host *host)
 837 {
 838         u32 reg;
 839         int err;
 840
 841         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 842         reg |= SDHCI_TEGRA_DLLCAL_CALIBRATE;
 843         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 844
 845         /* 1 ms sleep, 5 ms timeout */
 846         err = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_VENDOR_DLLCAL_STA,
 847                                  reg, !(reg & SDHCI_TEGRA_DLLCAL_STA_ACTIVE),
 848                                  1000, 5000);
 849         if (err)
 850                 dev_err(mmc_dev(host->mmc),
 851                         "HS400 delay line calibration timed out\n");
 852 }
 853
 854 static void tegra_sdhci_tap_correction(struct sdhci_host *host, u8 thd_up,
 855                                        u8 thd_low, u8 fixed_tap)
 856 {
 857         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 858         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 859         u32 val, tun_status;
 860         u8 word, bit, edge1, tap, window;
 861         bool tap_result;
 862         bool start_fail = false;
 863         bool start_pass = false;
 864         bool end_pass = false;
 865         bool first_fail = false;
 866         bool first_pass = false;
 867         u8 start_pass_tap = 0;
 868         u8 end_pass_tap = 0;
 869         u8 first_fail_tap = 0;
 870         u8 first_pass_tap = 0;
 871         u8 total_tuning_words = host->tuning_loop_count / TUNING_WORD_BIT_SIZE;
 872
 873         /*
 874          * Read auto-tuned results and extract good valid passing window by
 875          * filtering out un-wanted bubble/partial/merged windows.
 876          */
 877         for (word = 0; word < total_tuning_words; word++) {
 878                 val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
 879                 val &= ~SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK;
 880                 val |= word;
 881                 sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
 882                 tun_status = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS0);
 883                 bit = 0;
 884                 while (bit < TUNING_WORD_BIT_SIZE) {
 885                         tap = word * TUNING_WORD_BIT_SIZE + bit;
 886                         tap_result = tun_status & (1 << bit);
 887                         if (!tap_result && !start_fail) {
 888                                 start_fail = true;
 889                                 if (!first_fail) {
 890                                         first_fail_tap = tap;
 891                                         first_fail = true;
 892                                 }
 893
 894                         } else if (tap_result && start_fail && !start_pass) {
 895                                 start_pass_tap = tap;
 896                                 start_pass = true;
 897                                 if (!first_pass) {
 898                                         first_pass_tap = tap;
 899                                         first_pass = true;
 900                                 }
 901
 902                         } else if (!tap_result && start_fail && start_pass &&
 903                                    !end_pass) {
 904                                 end_pass_tap = tap - 1;
 905                                 end_pass = true;
 906                         } else if (tap_result && start_pass && start_fail &&
 907                                    end_pass) {
 908                                 window = end_pass_tap - start_pass_tap;
 909                                 /* discard merged window and bubble window */
 910                                 if (window >= thd_up || window < thd_low) {
 911                                         start_pass_tap = tap;
 912                                         end_pass = false;
 913                                 } else {
 914                                         /* set tap at middle of valid window */
 915                                         tap = start_pass_tap + window / 2;
 916                                         tegra_host->tuned_tap_delay = tap;
 917                                         return;
 918                                 }
 919                         }
 920
 921                         bit++;
 922                 }
 923         }
 924
 925         if (!first_fail) {
 926                 WARN(1, "no edge detected, continue with hw tuned delay.\n");
 927         } else if (first_pass) {
 928                 /* set tap location at fixed tap relative to the first edge */
 929                 edge1 = first_fail_tap + (first_pass_tap - first_fail_tap) / 2;
 930                 if (edge1 - 1 > fixed_tap)
 931                         tegra_host->tuned_tap_delay = edge1 - fixed_tap;
 932                 else
 933                         tegra_host->tuned_tap_delay = edge1 + fixed_tap;
 934         }
 935 }
 936
 937 static void tegra_sdhci_post_tuning(struct sdhci_host *host)
 938 {
 939         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 940         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 941         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 942         u32 avg_tap_dly, val, min_tap_dly, max_tap_dly;
 943         u8 fixed_tap, start_tap, end_tap, window_width;
 944         u8 thdupper, thdlower;
 945         u8 num_iter;
 946         u32 clk_rate_mhz, period_ps, bestcase, worstcase;
 947
 948         /* retain HW tuned tap to use incase if no correction is needed */
 949         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 950         tegra_host->tuned_tap_delay = (val & SDHCI_CLOCK_CTRL_TAP_MASK) >>
 951                                       SDHCI_CLOCK_CTRL_TAP_SHIFT;
 952         if (soc_data->min_tap_delay && soc_data->max_tap_delay) {
 953                 min_tap_dly = soc_data->min_tap_delay;
 954                 max_tap_dly = soc_data->max_tap_delay;
 955                 clk_rate_mhz = tegra_host->curr_clk_rate / USEC_PER_SEC;
 956                 period_ps = USEC_PER_SEC / clk_rate_mhz;
 957                 bestcase = period_ps / min_tap_dly;
 958                 worstcase = period_ps / max_tap_dly;
 959                 /*
 960                  * Upper and Lower bound thresholds used to detect merged and
 961                  * bubble windows
 962                  */
 963                 thdupper = (2 * worstcase + bestcase) / 2;
 964                 thdlower = worstcase / 4;
 965                 /*
 966                  * fixed tap is used when HW tuning result contains single edge
 967                  * and tap is set at fixed tap delay relative to the first edge
 968                  */
 969                 avg_tap_dly = (period_ps * 2) / (min_tap_dly + max_tap_dly);
 970                 fixed_tap = avg_tap_dly / 2;
 971
 972                 val = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS1);
 973                 start_tap = val & SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 974                 end_tap = (val >> SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT) &
 975                           SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 976                 window_width = end_tap - start_tap;
 977                 num_iter = host->tuning_loop_count;
 978                 /*
 979                  * partial window includes edges of the tuning range.
 980                  * merged window includes more taps so window width is higher
 981                  * than upper threshold.
 982                  */
 983                 if (start_tap == 0 || (end_tap == (num_iter - 1)) ||
 984                     (end_tap == num_iter - 2) || window_width >= thdupper) {
 985                         pr_debug("%s: Apply tuning correction\n",
 986                                  mmc_hostname(host->mmc));
 987                         tegra_sdhci_tap_correction(host, thdupper, thdlower,
 988                                                    fixed_tap);
 989                 }
 990         }
 991
 992         tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
 993 }
 994
 995 static int tegra_sdhci_execute_hw_tuning(struct mmc_host *mmc, u32 opcode)
 996 {
 997         struct sdhci_host *host = mmc_priv(mmc);
 998         int err;
 999
1000         err = sdhci_execute_tuning(mmc, opcode);
1001         if (!err && !host->tuning_err)
1002                 tegra_sdhci_post_tuning(host);
1003
1004         return err;
1005 }
1006
1007 static void tegra_sdhci_set_uhs_signaling(struct sdhci_host *host,
1008                                           unsigned timing)
1009 {
1010         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1011         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1012         bool set_default_tap = false;
1013         bool set_dqs_trim = false;
1014         bool do_hs400_dll_cal = false;
1015         u8 iter = TRIES_256;
1016         u32 val;
1017
1018         tegra_host->ddr_signaling = false;
1019         switch (timing) {
1020         case MMC_TIMING_UHS_SDR50:
1021                 break;
1022         case MMC_TIMING_UHS_SDR104:
1023         case MMC_TIMING_MMC_HS200:
1024                 /* Don't set default tap on tunable modes. */
1025                 iter = TRIES_128;
1026                 break;
1027         case MMC_TIMING_MMC_HS400:
1028                 set_dqs_trim = true;
1029                 do_hs400_dll_cal = true;
1030                 iter = TRIES_128;
1031                 break;
1032         case MMC_TIMING_MMC_DDR52:
1033         case MMC_TIMING_UHS_DDR50:
1034                 tegra_host->ddr_signaling = true;
1035                 set_default_tap = true;
1036                 break;
1037         default:
1038                 set_default_tap = true;
1039                 break;
1040         }
1041
1042         val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
1043         val &= ~(SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK |
1044                  SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK |
1045                  SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK);
1046         val |= (iter << SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT |
1047                 0 << SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT |
1048                 1 << SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT);
1049         sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
1050         sdhci_writel(host, 0, SDHCI_TEGRA_VNDR_TUN_CTRL1_0);
1051
1052         host->tuning_loop_count = (iter == TRIES_128) ? 128 : 256;
1053
1054         sdhci_set_uhs_signaling(host, timing);
1055
1056         tegra_sdhci_pad_autocalib(host);
1057
1058         if (tegra_host->tuned_tap_delay && !set_default_tap)
1059                 tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
1060         else
1061                 tegra_sdhci_set_tap(host, tegra_host->default_tap);
1062
1063         if (set_dqs_trim)
1064                 tegra_sdhci_set_dqs_trim(host, tegra_host->dqs_trim);
1065
1066         if (do_hs400_dll_cal)
1067                 tegra_sdhci_hs400_dll_cal(host);
1068 }
1069
1070 static int tegra_sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
1071 {
1072         unsigned int min, max;
1073
1074         /*
1075          * Start search for minimum tap value at 10, as smaller values are
1076          * may wrongly be reported as working but fail at higher speeds,
1077          * according to the TRM.
1078          */
1079         min = 10;
1080         while (min < 255) {
1081                 tegra_sdhci_set_tap(host, min);
1082                 if (!mmc_send_tuning(host->mmc, opcode, NULL))
1083                         break;
1084                 min++;
1085         }
1086
1087         /* Find the maximum tap value that still passes. */
1088         max = min + 1;
1089         while (max < 255) {
1090                 tegra_sdhci_set_tap(host, max);
1091                 if (mmc_send_tuning(host->mmc, opcode, NULL)) {
1092                         max--;
1093                         break;
1094                 }
1095                 max++;
1096         }
1097
1098         /* The TRM states the ideal tap value is at 75% in the passing range. */
1099         tegra_sdhci_set_tap(host, min + ((max - min) * 3 / 4));
1100
1101         return mmc_send_tuning(host->mmc, opcode, NULL);
1102 }
1103
1104 static int sdhci_tegra_start_signal_voltage_switch(struct mmc_host *mmc,
1105                                                    struct mmc_ios *ios)
1106 {
1107         struct sdhci_host *host = mmc_priv(mmc);
1108         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1109         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1110         int ret = 0;
1111
1112         if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
1113                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1114                 if (ret < 0)
1115                         return ret;
1116                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1117         } else if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180) {
1118                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1119                 if (ret < 0)
1120                         return ret;
1121                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1122         }
1123
1124         if (tegra_host->pad_calib_required)
1125                 tegra_sdhci_pad_autocalib(host);
1126
1127         return ret;
1128 }
1129
1130 static int tegra_sdhci_init_pinctrl_info(struct device *dev,
1131                                          struct sdhci_tegra *tegra_host)
1132 {
1133         tegra_host->pinctrl_sdmmc = devm_pinctrl_get(dev);
1134         if (IS_ERR(tegra_host->pinctrl_sdmmc)) {
1135                 dev_dbg(dev, "No pinctrl info, err: %ld\n",
1136                         PTR_ERR(tegra_host->pinctrl_sdmmc));
1137                 return -1;
1138         }
1139
1140         tegra_host->pinctrl_state_1v8_drv = pinctrl_lookup_state(
1141                                 tegra_host->pinctrl_sdmmc, "sdmmc-1v8-drv");
1142         if (IS_ERR(tegra_host->pinctrl_state_1v8_drv)) {
1143                 if (PTR_ERR(tegra_host->pinctrl_state_1v8_drv) == -ENODEV)
1144                         tegra_host->pinctrl_state_1v8_drv = NULL;
1145         }
1146
1147         tegra_host->pinctrl_state_3v3_drv = pinctrl_lookup_state(
1148                                 tegra_host->pinctrl_sdmmc, "sdmmc-3v3-drv");
1149         if (IS_ERR(tegra_host->pinctrl_state_3v3_drv)) {
1150                 if (PTR_ERR(tegra_host->pinctrl_state_3v3_drv) == -ENODEV)
1151                         tegra_host->pinctrl_state_3v3_drv = NULL;
1152         }
1153
1154         tegra_host->pinctrl_state_3v3 =
1155                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-3v3");
1156         if (IS_ERR(tegra_host->pinctrl_state_3v3)) {
1157                 dev_warn(dev, "Missing 3.3V pad state, err: %ld\n",
1158                          PTR_ERR(tegra_host->pinctrl_state_3v3));
1159                 return -1;
1160         }
1161
1162         tegra_host->pinctrl_state_1v8 =
1163                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-1v8");
1164         if (IS_ERR(tegra_host->pinctrl_state_1v8)) {
1165                 dev_warn(dev, "Missing 1.8V pad state, err: %ld\n",
1166                          PTR_ERR(tegra_host->pinctrl_state_1v8));
1167                 return -1;
1168         }
1169
1170         tegra_host->pad_control_available = true;
1171
1172         return 0;
1173 }
1174
1175 static void tegra_sdhci_voltage_switch(struct sdhci_host *host)
1176 {
1177         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1178         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1179         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1180
1181         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1182                 tegra_host->pad_calib_required = true;
1183 }
1184
1185 static void tegra_cqhci_writel(struct cqhci_host *cq_host, u32 val, int reg)
1186 {
1187         struct mmc_host *mmc = cq_host->mmc;
1188         struct sdhci_host *host = mmc_priv(mmc);
1189         u8 ctrl;
1190         ktime_t timeout;
1191         bool timed_out;
1192
1193         /*
1194          * During CQE resume/unhalt, CQHCI driver unhalts CQE prior to
1195          * cqhci_host_ops enable where SDHCI DMA and BLOCK_SIZE registers need
1196          * to be re-configured.
1197          * Tegra CQHCI/SDHCI prevents write access to block size register when
1198          * CQE is unhalted. So handling CQE resume sequence here to configure
1199          * SDHCI block registers prior to exiting CQE halt state.
1200          */
1201         if (reg == CQHCI_CTL && !(val & CQHCI_HALT) &&
1202             cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) {
1203                 sdhci_writew(host, SDHCI_TEGRA_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE);
1204                 sdhci_cqe_enable(mmc);
1205                 writel(val, cq_host->mmio + reg);
1206                 timeout = ktime_add_us(ktime_get(), 50);
1207                 while (1) {
1208                         timed_out = ktime_compare(ktime_get(), timeout) > 0;
1209                         ctrl = cqhci_readl(cq_host, CQHCI_CTL);
1210                         if (!(ctrl & CQHCI_HALT) || timed_out)
1211                                 break;
1212                 }
1213                 /*
1214                  * CQE usually resumes very quick, but incase if Tegra CQE
1215                  * doesn't resume retry unhalt.
1216                  */
1217                 if (timed_out)
1218                         writel(val, cq_host->mmio + reg);
1219         } else {
1220                 writel(val, cq_host->mmio + reg);
1221         }
1222 }
1223
1224 static void sdhci_tegra_update_dcmd_desc(struct mmc_host *mmc,
1225                                          struct mmc_request *mrq, u64 *data)
1226 {
1227         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(mmc_priv(mmc));
1228         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1229         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1230
1231         if (soc_data->nvquirks & NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING &&
1232             mrq->cmd->flags & MMC_RSP_R1B)
1233                 *data |= CQHCI_CMD_TIMING(1);
1234 }
1235
1236 static void sdhci_tegra_cqe_enable(struct mmc_host *mmc)
1237 {
1238         struct cqhci_host *cq_host = mmc->cqe_private;
1239         struct sdhci_host *host = mmc_priv(mmc);
1240         u32 val;
1241
1242         /*
1243          * Tegra CQHCI/SDMMC design prevents write access to sdhci block size
1244          * register when CQE is enabled and unhalted.
1245          * CQHCI driver enables CQE prior to activation, so disable CQE before
1246          * programming block size in sdhci controller and enable it back.
1247          */
1248         if (!cq_host->activated) {
1249                 val = cqhci_readl(cq_host, CQHCI_CFG);
1250                 if (val & CQHCI_ENABLE)
1251                         cqhci_writel(cq_host, (val & ~CQHCI_ENABLE),
1252                                      CQHCI_CFG);
1253                 sdhci_writew(host, SDHCI_TEGRA_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE);
1254                 sdhci_cqe_enable(mmc);
1255                 if (val & CQHCI_ENABLE)
1256                         cqhci_writel(cq_host, val, CQHCI_CFG);
1257         }
1258
1259         /*
1260          * CMD CRC errors are seen sometimes with some eMMC devices when status
1261          * command is sent during transfer of last data block which is the
1262          * default case as send status command block counter (CBC) is 1.
1263          * Recommended fix to set CBC to 0 allowing send status command only
1264          * when data lines are idle.
1265          */
1266         val = cqhci_readl(cq_host, CQHCI_SSC1);
1267         val &= ~CQHCI_SSC1_CBC_MASK;
1268         cqhci_writel(cq_host, val, CQHCI_SSC1);
1269 }
1270
1271 static void sdhci_tegra_dumpregs(struct mmc_host *mmc)
1272 {
1273         sdhci_dumpregs(mmc_priv(mmc));
1274 }
1275
1276 static u32 sdhci_tegra_cqhci_irq(struct sdhci_host *host, u32 intmask)
1277 {
1278         int cmd_error = 0;
1279         int data_error = 0;
1280
1281         if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
1282                 return intmask;
1283
1284         cqhci_irq(host->mmc, intmask, cmd_error, data_error);
1285
1286         return 0;
1287 }
1288
1289 static void tegra_sdhci_set_timeout(struct sdhci_host *host,
1290                                     struct mmc_command *cmd)
1291 {
1292         u32 val;
1293
1294         /*
1295          * HW busy detection timeout is based on programmed data timeout
1296          * counter and maximum supported timeout is 11s which may not be
1297          * enough for long operations like cache flush, sleep awake, erase.
1298          *
1299          * ERASE_TIMEOUT_LIMIT bit of VENDOR_MISC_CTRL register allows
1300          * host controller to wait for busy state until the card is busy
1301          * without HW timeout.
1302          *
1303          * So, use infinite busy wait mode for operations that may take
1304          * more than maximum HW busy timeout of 11s otherwise use finite
1305          * busy wait mode.
1306          */
1307         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_MISC_CTRL);
1308         if (cmd && cmd->busy_timeout >= 11 * MSEC_PER_SEC)
1309                 val |= SDHCI_MISC_CTRL_ERASE_TIMEOUT_LIMIT;
1310         else
1311                 val &= ~SDHCI_MISC_CTRL_ERASE_TIMEOUT_LIMIT;
1312         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_MISC_CTRL);
1313
1314         __sdhci_set_timeout(host, cmd);
1315 }
1316
1317 static void sdhci_tegra_cqe_pre_enable(struct mmc_host *mmc)
1318 {
1319         struct cqhci_host *cq_host = mmc->cqe_private;
1320         u32 reg;
1321
1322         reg = cqhci_readl(cq_host, CQHCI_CFG);
1323         reg |= CQHCI_ENABLE;
1324         cqhci_writel(cq_host, reg, CQHCI_CFG);
1325 }
1326
1327 static void sdhci_tegra_cqe_post_disable(struct mmc_host *mmc)
1328 {
1329         struct cqhci_host *cq_host = mmc->cqe_private;
1330         struct sdhci_host *host = mmc_priv(mmc);
1331         u32 reg;
1332
1333         reg = cqhci_readl(cq_host, CQHCI_CFG);
1334         reg &= ~CQHCI_ENABLE;
1335         cqhci_writel(cq_host, reg, CQHCI_CFG);
1336         sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1337 }
1338
1339 static const struct cqhci_host_ops sdhci_tegra_cqhci_ops = {
1340         .write_l    = tegra_cqhci_writel,
1341         .enable = sdhci_tegra_cqe_enable,
1342         .disable = sdhci_cqe_disable,
1343         .dumpregs = sdhci_tegra_dumpregs,
1344         .update_dcmd_desc = sdhci_tegra_update_dcmd_desc,
1345         .pre_enable = sdhci_tegra_cqe_pre_enable,
1346         .post_disable = sdhci_tegra_cqe_post_disable,
1347 };
1348
1349 static int tegra_sdhci_set_dma_mask(struct sdhci_host *host)
1350 {
1351         struct sdhci_pltfm_host *platform = sdhci_priv(host);
1352         struct sdhci_tegra *tegra = sdhci_pltfm_priv(platform);
1353         const struct sdhci_tegra_soc_data *soc = tegra->soc_data;
1354         struct device *dev = mmc_dev(host->mmc);
1355
1356         if (soc->dma_mask)
1357                 return dma_set_mask_and_coherent(dev, soc->dma_mask);
1358
1359         return 0;
1360 }
1361
1362 static const struct sdhci_ops tegra_sdhci_ops = {
1363         .get_ro     = tegra_sdhci_get_ro,
1364         .read_w     = tegra_sdhci_readw,
1365         .write_l    = tegra_sdhci_writel,
1366         .set_clock  = tegra_sdhci_set_clock,
1367         .set_dma_mask = tegra_sdhci_set_dma_mask,
1368         .set_bus_width = sdhci_set_bus_width,
1369         .reset      = tegra_sdhci_reset,
1370         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1371         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1372         .voltage_switch = tegra_sdhci_voltage_switch,
1373         .get_max_clock = tegra_sdhci_get_max_clock,
1374 };
1375
1376 static const struct sdhci_pltfm_data sdhci_tegra20_pdata = {
1377         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1378                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1379                   SDHCI_QUIRK_NO_HISPD_BIT |
1380                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1381                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1382         .ops  = &tegra_sdhci_ops,
1383 };
1384
1385 static const struct sdhci_tegra_soc_data soc_data_tegra20 = {
1386         .pdata = &sdhci_tegra20_pdata,
1387         .dma_mask = DMA_BIT_MASK(32),
1388         .nvquirks = NVQUIRK_FORCE_SDHCI_SPEC_200 |
1389                     NVQUIRK_HAS_ANDROID_GPT_SECTOR |
1390                     NVQUIRK_ENABLE_BLOCK_GAP_DET,
1391 };
1392
1393 static const struct sdhci_pltfm_data sdhci_tegra30_pdata = {
1394         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1395                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1396                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1397                   SDHCI_QUIRK_NO_HISPD_BIT |
1398                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1399                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1400         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1401                    SDHCI_QUIRK2_BROKEN_HS200 |
1402                    /*
1403                     * Auto-CMD23 leads to "Got command interrupt 0x00010000 even
1404                     * though no command operation was in progress."
1405                     *
1406                     * The exact reason is unknown, as the same hardware seems
1407                     * to support Auto CMD23 on a downstream 3.1 kernel.
1408                     */
1409                    SDHCI_QUIRK2_ACMD23_BROKEN,
1410         .ops  = &tegra_sdhci_ops,
1411 };
1412
1413 static const struct sdhci_tegra_soc_data soc_data_tegra30 = {
1414         .pdata = &sdhci_tegra30_pdata,
1415         .dma_mask = DMA_BIT_MASK(32),
1416         .nvquirks = NVQUIRK_ENABLE_SDHCI_SPEC_300 |
1417                     NVQUIRK_ENABLE_SDR50 |
1418                     NVQUIRK_ENABLE_SDR104 |
1419                     NVQUIRK_HAS_ANDROID_GPT_SECTOR |
1420                     NVQUIRK_HAS_PADCALIB,
1421 };
1422
1423 static const struct sdhci_ops tegra114_sdhci_ops = {
1424         .get_ro     = tegra_sdhci_get_ro,
1425         .read_w     = tegra_sdhci_readw,
1426         .write_w    = tegra_sdhci_writew,
1427         .write_l    = tegra_sdhci_writel,
1428         .set_clock  = tegra_sdhci_set_clock,
1429         .set_dma_mask = tegra_sdhci_set_dma_mask,
1430         .set_bus_width = sdhci_set_bus_width,
1431         .reset      = tegra_sdhci_reset,
1432         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1433         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1434         .voltage_switch = tegra_sdhci_voltage_switch,
1435         .get_max_clock = tegra_sdhci_get_max_clock,
1436 };
1437
1438 static const struct sdhci_pltfm_data sdhci_tegra114_pdata = {
1439         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1440                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1441                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1442                   SDHCI_QUIRK_NO_HISPD_BIT |
1443                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1444                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1445         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1446         .ops  = &tegra114_sdhci_ops,
1447 };
1448
1449 static const struct sdhci_tegra_soc_data soc_data_tegra114 = {
1450         .pdata = &sdhci_tegra114_pdata,
1451         .dma_mask = DMA_BIT_MASK(32),
1452         .nvquirks = NVQUIRK_HAS_ANDROID_GPT_SECTOR,
1453 };
1454
1455 static const struct sdhci_pltfm_data sdhci_tegra124_pdata = {
1456         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1457                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1458                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1459                   SDHCI_QUIRK_NO_HISPD_BIT |
1460                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1461                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1462         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1463         .ops  = &tegra114_sdhci_ops,
1464 };
1465
1466 static const struct sdhci_tegra_soc_data soc_data_tegra124 = {
1467         .pdata = &sdhci_tegra124_pdata,
1468         .dma_mask = DMA_BIT_MASK(34),
1469         .nvquirks = NVQUIRK_HAS_ANDROID_GPT_SECTOR,
1470 };
1471
1472 static const struct sdhci_ops tegra210_sdhci_ops = {
1473         .get_ro     = tegra_sdhci_get_ro,
1474         .read_w     = tegra_sdhci_readw,
1475         .write_w    = tegra210_sdhci_writew,
1476         .write_l    = tegra_sdhci_writel,
1477         .set_clock  = tegra_sdhci_set_clock,
1478         .set_dma_mask = tegra_sdhci_set_dma_mask,
1479         .set_bus_width = sdhci_set_bus_width,
1480         .reset      = tegra_sdhci_reset,
1481         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1482         .voltage_switch = tegra_sdhci_voltage_switch,
1483         .get_max_clock = tegra_sdhci_get_max_clock,
1484         .set_timeout = tegra_sdhci_set_timeout,
1485 };
1486
1487 static const struct sdhci_pltfm_data sdhci_tegra210_pdata = {
1488         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1489                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1490                   SDHCI_QUIRK_NO_HISPD_BIT |
1491                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1492                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1493         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1494         .ops  = &tegra210_sdhci_ops,
1495 };
1496
1497 static const struct sdhci_tegra_soc_data soc_data_tegra210 = {
1498         .pdata = &sdhci_tegra210_pdata,
1499         .dma_mask = DMA_BIT_MASK(34),
1500         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1501                     NVQUIRK_HAS_PADCALIB |
1502                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1503                     NVQUIRK_ENABLE_SDR50 |
1504                     NVQUIRK_ENABLE_SDR104 |
1505                     NVQUIRK_HAS_TMCLK,
1506         .min_tap_delay = 106,
1507         .max_tap_delay = 185,
1508 };
1509
1510 static const struct sdhci_ops tegra186_sdhci_ops = {
1511         .get_ro     = tegra_sdhci_get_ro,
1512         .read_w     = tegra_sdhci_readw,
1513         .write_l    = tegra_sdhci_writel,
1514         .set_clock  = tegra_sdhci_set_clock,
1515         .set_dma_mask = tegra_sdhci_set_dma_mask,
1516         .set_bus_width = sdhci_set_bus_width,
1517         .reset      = tegra_sdhci_reset,
1518         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1519         .voltage_switch = tegra_sdhci_voltage_switch,
1520         .get_max_clock = tegra_sdhci_get_max_clock,
1521         .irq = sdhci_tegra_cqhci_irq,
1522         .set_timeout = tegra_sdhci_set_timeout,
1523 };
1524
1525 static const struct sdhci_pltfm_data sdhci_tegra186_pdata = {
1526         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1527                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1528                   SDHCI_QUIRK_NO_HISPD_BIT |
1529                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1530                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1531         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1532                    SDHCI_QUIRK2_ISSUE_CMD_DAT_RESET_TOGETHER,
1533         .ops  = &tegra186_sdhci_ops,
1534 };
1535
1536 static const struct sdhci_tegra_soc_data soc_data_tegra186 = {
1537         .pdata = &sdhci_tegra186_pdata,
1538         .dma_mask = DMA_BIT_MASK(40),
1539         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1540                     NVQUIRK_HAS_PADCALIB |
1541                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1542                     NVQUIRK_ENABLE_SDR50 |
1543                     NVQUIRK_ENABLE_SDR104 |
1544                     NVQUIRK_HAS_TMCLK |
1545                     NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING,
1546         .min_tap_delay = 84,
1547         .max_tap_delay = 136,
1548 };
1549
1550 static const struct sdhci_tegra_soc_data soc_data_tegra194 = {
1551         .pdata = &sdhci_tegra186_pdata,
1552         .dma_mask = DMA_BIT_MASK(39),
1553         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1554                     NVQUIRK_HAS_PADCALIB |
1555                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1556                     NVQUIRK_ENABLE_SDR50 |
1557                     NVQUIRK_ENABLE_SDR104 |
1558                     NVQUIRK_HAS_TMCLK,
1559         .min_tap_delay = 96,
1560         .max_tap_delay = 139,
1561 };
1562
1563 static const struct sdhci_tegra_soc_data soc_data_tegra234 = {
1564         .pdata = &sdhci_tegra186_pdata,
1565         .dma_mask = DMA_BIT_MASK(39),
1566         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1567                     NVQUIRK_HAS_PADCALIB |
1568                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1569                     NVQUIRK_ENABLE_SDR50 |
1570                     NVQUIRK_ENABLE_SDR104 |
1571                     NVQUIRK_PROGRAM_STREAMID |
1572                     NVQUIRK_HAS_TMCLK,
1573         .min_tap_delay = 95,
1574         .max_tap_delay = 111,
1575 };
1576
1577 static const struct of_device_id sdhci_tegra_dt_match[] = {
1578         { .compatible = "nvidia,tegra234-sdhci", .data = &soc_data_tegra234 },
1579         { .compatible = "nvidia,tegra194-sdhci", .data = &soc_data_tegra194 },
1580         { .compatible = "nvidia,tegra186-sdhci", .data = &soc_data_tegra186 },
1581         { .compatible = "nvidia,tegra210-sdhci", .data = &soc_data_tegra210 },
1582         { .compatible = "nvidia,tegra124-sdhci", .data = &soc_data_tegra124 },
1583         { .compatible = "nvidia,tegra114-sdhci", .data = &soc_data_tegra114 },
1584         { .compatible = "nvidia,tegra30-sdhci", .data = &soc_data_tegra30 },
1585         { .compatible = "nvidia,tegra20-sdhci", .data = &soc_data_tegra20 },
1586         {}
1587 };
1588 MODULE_DEVICE_TABLE(of, sdhci_tegra_dt_match);
1589
1590 static int sdhci_tegra_add_host(struct sdhci_host *host)
1591 {
1592         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1593         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1594         struct cqhci_host *cq_host;
1595         bool dma64;
1596         int ret;
1597
1598         if (!tegra_host->enable_hwcq)
1599                 return sdhci_add_host(host);
1600
1601         sdhci_enable_v4_mode(host);
1602
1603         ret = sdhci_setup_host(host);
1604         if (ret)
1605                 return ret;
1606
1607         host->mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
1608
1609         cq_host = devm_kzalloc(mmc_dev(host->mmc),
1610                                 sizeof(*cq_host), GFP_KERNEL);
1611         if (!cq_host) {
1612                 ret = -ENOMEM;
1613                 goto cleanup;
1614         }
1615
1616         cq_host->mmio = host->ioaddr + SDHCI_TEGRA_CQE_BASE_ADDR;
1617         cq_host->ops = &sdhci_tegra_cqhci_ops;
1618
1619         dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
1620         if (dma64)
1621                 cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
1622
1623         ret = cqhci_init(cq_host, host->mmc, dma64);
1624         if (ret)
1625                 goto cleanup;
1626
1627         ret = __sdhci_add_host(host);
1628         if (ret)
1629                 goto cleanup;
1630
1631         return 0;
1632
1633 cleanup:
1634         sdhci_cleanup_host(host);
1635         return ret;
1636 }
1637
1638 /* Program MC streamID for DMA transfers */
1639 static void sdhci_tegra_program_stream_id(struct sdhci_host *host)
1640 {
1641         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1642         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1643
1644         if (tegra_host->soc_data->nvquirks & NVQUIRK_PROGRAM_STREAMID) {
1645                 tegra_sdhci_writel(host, FIELD_PREP(GENMASK(15, 8), tegra_host->stream_id) |
1646                                          FIELD_PREP(GENMASK(7, 0), tegra_host->stream_id),
1647                                          SDHCI_TEGRA_CIF2AXI_CTRL_0);
1648         }
1649 }
1650
1651 static int sdhci_tegra_probe(struct platform_device *pdev)
1652 {
1653         const struct sdhci_tegra_soc_data *soc_data;
1654         struct sdhci_host *host;
1655         struct sdhci_pltfm_host *pltfm_host;
1656         struct sdhci_tegra *tegra_host;
1657         struct clk *clk;
1658         int rc;
1659
1660         soc_data = of_device_get_match_data(&pdev->dev);
1661         if (!soc_data)
1662                 return -EINVAL;
1663
1664         host = sdhci_pltfm_init(pdev, soc_data->pdata, sizeof(*tegra_host));
1665         if (IS_ERR(host))
1666                 return PTR_ERR(host);
1667         pltfm_host = sdhci_priv(host);
1668
1669         tegra_host = sdhci_pltfm_priv(pltfm_host);
1670         tegra_host->ddr_signaling = false;
1671         tegra_host->pad_calib_required = false;
1672         tegra_host->pad_control_available = false;
1673         tegra_host->soc_data = soc_data;
1674
1675         if (soc_data->nvquirks & NVQUIRK_HAS_ANDROID_GPT_SECTOR)
1676                 host->mmc->caps2 |= MMC_CAP2_ALT_GPT_TEGRA;
1677
1678         if (soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL) {
1679                 rc = tegra_sdhci_init_pinctrl_info(&pdev->dev, tegra_host);
1680                 if (rc == 0)
1681                         host->mmc_host_ops.start_signal_voltage_switch =
1682                                 sdhci_tegra_start_signal_voltage_switch;
1683         }
1684
1685         /* Hook to periodically rerun pad calibration */
1686         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1687                 host->mmc_host_ops.request = tegra_sdhci_request;
1688
1689         host->mmc_host_ops.hs400_enhanced_strobe =
1690                         tegra_sdhci_hs400_enhanced_strobe;
1691
1692         if (!host->ops->platform_execute_tuning)
1693                 host->mmc_host_ops.execute_tuning =
1694                                 tegra_sdhci_execute_hw_tuning;
1695
1696         rc = mmc_of_parse(host->mmc);
1697         if (rc)
1698                 goto err_parse_dt;
1699
1700         if (tegra_host->soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
1701                 host->mmc->caps |= MMC_CAP_1_8V_DDR;
1702
1703         /* HW busy detection is supported, but R1B responses are required. */
1704         host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_NEED_RSP_BUSY;
1705
1706         /* GPIO CD can be set as a wakeup source */
1707         host->mmc->caps |= MMC_CAP_CD_WAKE;
1708
1709         tegra_sdhci_parse_dt(host);
1710
1711         if (tegra_host->soc_data->nvquirks & NVQUIRK_PROGRAM_STREAMID &&
1712             !tegra_dev_iommu_get_stream_id(&pdev->dev, &tegra_host->stream_id)) {
1713                 dev_warn(mmc_dev(host->mmc), "missing IOMMU stream ID\n");
1714                 tegra_host->stream_id = 0x7f;
1715         }
1716
1717         tegra_host->power_gpio = devm_gpiod_get_optional(&pdev->dev, "power",
1718                                                          GPIOD_OUT_HIGH);
1719         if (IS_ERR(tegra_host->power_gpio)) {
1720                 rc = PTR_ERR(tegra_host->power_gpio);
1721                 goto err_power_req;
1722         }
1723
1724         /*
1725          * Tegra210 has a separate SDMMC_LEGACY_TM clock used for host
1726          * timeout clock and SW can choose TMCLK or SDCLK for hardware
1727          * data timeout through the bit USE_TMCLK_FOR_DATA_TIMEOUT of
1728          * the register SDHCI_TEGRA_VENDOR_SYS_SW_CTRL.
1729          *
1730          * USE_TMCLK_FOR_DATA_TIMEOUT bit default is set to 1 and SDMMC uses
1731          * 12Mhz TMCLK which is advertised in host capability register.
1732          * With TMCLK of 12Mhz provides maximum data timeout period that can
1733          * be achieved is 11s better than using SDCLK for data timeout.
1734          *
1735          * So, TMCLK is set to 12Mhz and kept enabled all the time on SoC's
1736          * supporting separate TMCLK.
1737          */
1738
1739         if (soc_data->nvquirks & NVQUIRK_HAS_TMCLK) {
1740                 clk = devm_clk_get(&pdev->dev, "tmclk");
1741                 if (IS_ERR(clk)) {
1742                         rc = PTR_ERR(clk);
1743                         if (rc == -EPROBE_DEFER)
1744                                 goto err_power_req;
1745
1746                         dev_warn(&pdev->dev, "failed to get tmclk: %d\n", rc);
1747                         clk = NULL;
1748                 }
1749
1750                 clk_set_rate(clk, 12000000);
1751                 rc = clk_prepare_enable(clk);
1752                 if (rc) {
1753                         dev_err(&pdev->dev,
1754                                 "failed to enable tmclk: %d\n", rc);
1755                         goto err_power_req;
1756                 }
1757
1758                 tegra_host->tmclk = clk;
1759         }
1760
1761         clk = devm_clk_get(mmc_dev(host->mmc), NULL);
1762         if (IS_ERR(clk)) {
1763                 rc = dev_err_probe(&pdev->dev, PTR_ERR(clk),
1764                                    "failed to get clock\n");
1765                 goto err_clk_get;
1766         }
1767         pltfm_host->clk = clk;
1768
1769         tegra_host->rst = devm_reset_control_get_exclusive(&pdev->dev,
1770                                                            "sdhci");
1771         if (IS_ERR(tegra_host->rst)) {
1772                 rc = PTR_ERR(tegra_host->rst);
1773                 dev_err(&pdev->dev, "failed to get reset control: %d\n", rc);
1774                 goto err_rst_get;
1775         }
1776
1777         rc = devm_tegra_core_dev_init_opp_table_common(&pdev->dev);
1778         if (rc)
1779                 goto err_rst_get;
1780
1781         pm_runtime_enable(&pdev->dev);
1782         rc = pm_runtime_resume_and_get(&pdev->dev);
1783         if (rc)
1784                 goto err_pm_get;
1785
1786         rc = reset_control_assert(tegra_host->rst);
1787         if (rc)
1788                 goto err_rst_assert;
1789
1790         usleep_range(2000, 4000);
1791
1792         rc = reset_control_deassert(tegra_host->rst);
1793         if (rc)
1794                 goto err_rst_assert;
1795
1796         usleep_range(2000, 4000);
1797
1798         rc = sdhci_tegra_add_host(host);
1799         if (rc)
1800                 goto err_add_host;
1801
1802         sdhci_tegra_program_stream_id(host);
1803
1804         return 0;
1805
1806 err_add_host:
1807         reset_control_assert(tegra_host->rst);
1808 err_rst_assert:
1809         pm_runtime_put_sync_suspend(&pdev->dev);
1810 err_pm_get:
1811         pm_runtime_disable(&pdev->dev);
1812 err_rst_get:
1813 err_clk_get:
1814         clk_disable_unprepare(tegra_host->tmclk);
1815 err_power_req:
1816 err_parse_dt:
1817         sdhci_pltfm_free(pdev);
1818         return rc;
1819 }
1820
1821 static int sdhci_tegra_remove(struct platform_device *pdev)
1822 {
1823         struct sdhci_host *host = platform_get_drvdata(pdev);
1824         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1825         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1826
1827         sdhci_remove_host(host, 0);
1828
1829         reset_control_assert(tegra_host->rst);
1830         usleep_range(2000, 4000);
1831
1832         pm_runtime_put_sync_suspend(&pdev->dev);
1833         pm_runtime_force_suspend(&pdev->dev);
1834
1835         clk_disable_unprepare(tegra_host->tmclk);
1836         sdhci_pltfm_free(pdev);
1837
1838         return 0;
1839 }
1840
1841 static int __maybe_unused sdhci_tegra_runtime_suspend(struct device *dev)
1842 {
1843         struct sdhci_host *host = dev_get_drvdata(dev);
1844         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1845
1846         clk_disable_unprepare(pltfm_host->clk);
1847
1848         return 0;
1849 }
1850
1851 static int __maybe_unused sdhci_tegra_runtime_resume(struct device *dev)
1852 {
1853         struct sdhci_host *host = dev_get_drvdata(dev);
1854         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1855
1856         return clk_prepare_enable(pltfm_host->clk);
1857 }
1858
1859 #ifdef CONFIG_PM_SLEEP
1860 static int sdhci_tegra_suspend(struct device *dev)
1861 {
1862         struct sdhci_host *host = dev_get_drvdata(dev);
1863         int ret;
1864
1865         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1866                 ret = cqhci_suspend(host->mmc);
1867                 if (ret)
1868                         return ret;
1869         }
1870
1871         ret = sdhci_suspend_host(host);
1872         if (ret) {
1873                 cqhci_resume(host->mmc);
1874                 return ret;
1875         }
1876
1877         ret = pm_runtime_force_suspend(dev);
1878         if (ret) {
1879                 sdhci_resume_host(host);
1880                 cqhci_resume(host->mmc);
1881                 return ret;
1882         }
1883
1884         return mmc_gpio_set_cd_wake(host->mmc, true);
1885 }
1886
1887 static int sdhci_tegra_resume(struct device *dev)
1888 {
1889         struct sdhci_host *host = dev_get_drvdata(dev);
1890         int ret;
1891
1892         ret = mmc_gpio_set_cd_wake(host->mmc, false);
1893         if (ret)
1894                 return ret;
1895
1896         ret = pm_runtime_force_resume(dev);
1897         if (ret)
1898                 return ret;
1899
1900         sdhci_tegra_program_stream_id(host);
1901
1902         ret = sdhci_resume_host(host);
1903         if (ret)
1904                 goto disable_clk;
1905
1906         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1907                 ret = cqhci_resume(host->mmc);
1908                 if (ret)
1909                         goto suspend_host;
1910         }
1911
1912         return 0;
1913
1914 suspend_host:
1915         sdhci_suspend_host(host);
1916 disable_clk:
1917         pm_runtime_force_suspend(dev);
1918         return ret;
1919 }
1920 #endif
1921
1922 static const struct dev_pm_ops sdhci_tegra_dev_pm_ops = {
1923         SET_RUNTIME_PM_OPS(sdhci_tegra_runtime_suspend, sdhci_tegra_runtime_resume,
1924                            NULL)
1925         SET_SYSTEM_SLEEP_PM_OPS(sdhci_tegra_suspend, sdhci_tegra_resume)
1926 };
1927
1928 static struct platform_driver sdhci_tegra_driver = {
1929         .driver         = {
1930                 .name   = "sdhci-tegra",
1931                 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
1932                 .of_match_table = sdhci_tegra_dt_match,
1933                 .pm     = &sdhci_tegra_dev_pm_ops,
1934         },
1935         .probe          = sdhci_tegra_probe,
1936         .remove         = sdhci_tegra_remove,
1937 };
1938
1939 module_platform_driver(sdhci_tegra_driver);
1940
1941 MODULE_DESCRIPTION("SDHCI driver for Tegra");
1942 MODULE_AUTHOR("Google, Inc.");
1943 MODULE_LICENSE("GPL v2");