drivers/spi/spi-imx.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
   3 // Copyright (C) 2008 Juergen Beisert
   4
   5 #include <linux/clk.h>
   6 #include <linux/completion.h>
   7 #include <linux/delay.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/err.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/io.h>
  13 #include <linux/irq.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/pinctrl/consumer.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/pm_runtime.h>
  19 #include <linux/slab.h>
  20 #include <linux/spi/spi.h>
  21 #include <linux/types.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/property.h>
  25
  26 #include <linux/dma/imx-dma.h>
  27
  28 #define DRIVER_NAME "spi_imx"
  29
  30 static bool use_dma = true;
  31 module_param(use_dma, bool, 0644);
  32 MODULE_PARM_DESC(use_dma, "Enable usage of DMA when available (default)");
  33
  34 /* define polling limits */
  35 static unsigned int polling_limit_us = 30;
  36 module_param(polling_limit_us, uint, 0664);
  37 MODULE_PARM_DESC(polling_limit_us,
  38                  "time in us to run a transfer in polling mode\n");
  39
  40 #define MXC_RPM_TIMEOUT         2000 /* 2000ms */
  41
  42 #define MXC_CSPIRXDATA          0x00
  43 #define MXC_CSPITXDATA          0x04
  44 #define MXC_CSPICTRL            0x08
  45 #define MXC_CSPIINT             0x0c
  46 #define MXC_RESET               0x1c
  47
  48 /* generic defines to abstract from the different register layouts */
  49 #define MXC_INT_RR      (1 << 0) /* Receive data ready interrupt */
  50 #define MXC_INT_TE      (1 << 1) /* Transmit FIFO empty interrupt */
  51 #define MXC_INT_RDR     BIT(4) /* Receive date threshold interrupt */
  52
  53 /* The maximum bytes that a sdma BD can transfer. */
  54 #define MAX_SDMA_BD_BYTES (1 << 15)
  55 #define MX51_ECSPI_CTRL_MAX_BURST       512
  56 /* The maximum bytes that IMX53_ECSPI can transfer in slave mode.*/
  57 #define MX53_MAX_TRANSFER_BYTES         512
  58
  59 enum spi_imx_devtype {
  60         IMX1_CSPI,
  61         IMX21_CSPI,
  62         IMX27_CSPI,
  63         IMX31_CSPI,
  64         IMX35_CSPI,     /* CSPI on all i.mx except above */
  65         IMX51_ECSPI,    /* ECSPI on i.mx51 */
  66         IMX53_ECSPI,    /* ECSPI on i.mx53 and later */
  67 };
  68
  69 struct spi_imx_data;
  70
  71 struct spi_imx_devtype_data {
  72         void (*intctrl)(struct spi_imx_data *spi_imx, int enable);
  73         int (*prepare_message)(struct spi_imx_data *spi_imx, struct spi_message *msg);
  74         int (*prepare_transfer)(struct spi_imx_data *spi_imx, struct spi_device *spi);
  75         void (*trigger)(struct spi_imx_data *spi_imx);
  76         int (*rx_available)(struct spi_imx_data *spi_imx);
  77         void (*reset)(struct spi_imx_data *spi_imx);
  78         void (*setup_wml)(struct spi_imx_data *spi_imx);
  79         void (*disable)(struct spi_imx_data *spi_imx);
  80         bool has_dmamode;
  81         bool has_slavemode;
  82         unsigned int fifo_size;
  83         bool dynamic_burst;
  84         /*
  85          * ERR009165 fixed or not:
  86          * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
  87          */
  88         bool tx_glitch_fixed;
  89         enum spi_imx_devtype devtype;
  90 };
  91
  92 struct spi_imx_data {
  93         struct spi_controller *controller;
  94         struct device *dev;
  95
  96         struct completion xfer_done;
  97         void __iomem *base;
  98         unsigned long base_phys;
  99
 100         struct clk *clk_per;
 101         struct clk *clk_ipg;
 102         unsigned long spi_clk;
 103         unsigned int spi_bus_clk;
 104
 105         unsigned int bits_per_word;
 106         unsigned int spi_drctl;
 107
 108         unsigned int count, remainder;
 109         void (*tx)(struct spi_imx_data *spi_imx);
 110         void (*rx)(struct spi_imx_data *spi_imx);
 111         void *rx_buf;
 112         const void *tx_buf;
 113         unsigned int txfifo; /* number of words pushed in tx FIFO */
 114         unsigned int dynamic_burst;
 115         bool rx_only;
 116
 117         /* Slave mode */
 118         bool slave_mode;
 119         bool slave_aborted;
 120         unsigned int slave_burst;
 121
 122         /* DMA */
 123         bool usedma;
 124         u32 wml;
 125         struct completion dma_rx_completion;
 126         struct completion dma_tx_completion;
 127
 128         const struct spi_imx_devtype_data *devtype_data;
 129 };
 130
 131 static inline int is_imx27_cspi(struct spi_imx_data *d)
 132 {
 133         return d->devtype_data->devtype == IMX27_CSPI;
 134 }
 135
 136 static inline int is_imx35_cspi(struct spi_imx_data *d)
 137 {
 138         return d->devtype_data->devtype == IMX35_CSPI;
 139 }
 140
 141 static inline int is_imx51_ecspi(struct spi_imx_data *d)
 142 {
 143         return d->devtype_data->devtype == IMX51_ECSPI;
 144 }
 145
 146 static inline int is_imx53_ecspi(struct spi_imx_data *d)
 147 {
 148         return d->devtype_data->devtype == IMX53_ECSPI;
 149 }
 150
 151 #define MXC_SPI_BUF_RX(type)                                            \
 152 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)         \
 153 {                                                                       \
 154         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);       \
 155                                                                         \
 156         if (spi_imx->rx_buf) {                                          \
 157                 *(type *)spi_imx->rx_buf = val;                         \
 158                 spi_imx->rx_buf += sizeof(type);                        \
 159         }                                                               \
 160                                                                         \
 161         spi_imx->remainder -= sizeof(type);                             \
 162 }
 163
 164 #define MXC_SPI_BUF_TX(type)                                            \
 165 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)         \
 166 {                                                                       \
 167         type val = 0;                                                   \
 168                                                                         \
 169         if (spi_imx->tx_buf) {                                          \
 170                 val = *(type *)spi_imx->tx_buf;                         \
 171                 spi_imx->tx_buf += sizeof(type);                        \
 172         }                                                               \
 173                                                                         \
 174         spi_imx->count -= sizeof(type);                                 \
 175                                                                         \
 176         writel(val, spi_imx->base + MXC_CSPITXDATA);                    \
 177 }
 178
 179 MXC_SPI_BUF_RX(u8)
 180 MXC_SPI_BUF_TX(u8)
 181 MXC_SPI_BUF_RX(u16)
 182 MXC_SPI_BUF_TX(u16)
 183 MXC_SPI_BUF_RX(u32)
 184 MXC_SPI_BUF_TX(u32)
 185
 186 /* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
 187  * (which is currently not the case in this driver)
 188  */
 189 static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
 190         256, 384, 512, 768, 1024};
 191
 192 /* MX21, MX27 */
 193 static unsigned int spi_imx_clkdiv_1(unsigned int fin,
 194                 unsigned int fspi, unsigned int max, unsigned int *fres)
 195 {
 196         int i;
 197
 198         for (i = 2; i < max; i++)
 199                 if (fspi * mxc_clkdivs[i] >= fin)
 200                         break;
 201
 202         *fres = fin / mxc_clkdivs[i];
 203         return i;
 204 }
 205
 206 /* MX1, MX31, MX35, MX51 CSPI */
 207 static unsigned int spi_imx_clkdiv_2(unsigned int fin,
 208                 unsigned int fspi, unsigned int *fres)
 209 {
 210         int i, div = 4;
 211
 212         for (i = 0; i < 7; i++) {
 213                 if (fspi * div >= fin)
 214                         goto out;
 215                 div <<= 1;
 216         }
 217
 218 out:
 219         *fres = fin / div;
 220         return i;
 221 }
 222
 223 static int spi_imx_bytes_per_word(const int bits_per_word)
 224 {
 225         if (bits_per_word <= 8)
 226                 return 1;
 227         else if (bits_per_word <= 16)
 228                 return 2;
 229         else
 230                 return 4;
 231 }
 232
 233 static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device *spi,
 234                          struct spi_transfer *transfer)
 235 {
 236         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
 237
 238         if (!use_dma || controller->fallback)
 239                 return false;
 240
 241         if (!controller->dma_rx)
 242                 return false;
 243
 244         if (spi_imx->slave_mode)
 245                 return false;
 246
 247         if (transfer->len < spi_imx->devtype_data->fifo_size)
 248                 return false;
 249
 250         spi_imx->dynamic_burst = 0;
 251
 252         return true;
 253 }
 254
 255 /*
 256  * Note the number of natively supported chip selects for MX51 is 4. Some
 257  * devices may have less actual SS pins but the register map supports 4. When
 258  * using gpio chip selects the cs values passed into the macros below can go
 259  * outside the range 0 - 3. We therefore need to limit the cs value to avoid
 260  * corrupting bits outside the allocated locations.
 261  *
 262  * The simplest way to do this is to just mask the cs bits to 2 bits. This
 263  * still allows all 4 native chip selects to work as well as gpio chip selects
 264  * (which can use any of the 4 chip select configurations).
 265  */
 266
 267 #define MX51_ECSPI_CTRL         0x08
 268 #define MX51_ECSPI_CTRL_ENABLE          (1 <<  0)
 269 #define MX51_ECSPI_CTRL_XCH             (1 <<  2)
 270 #define MX51_ECSPI_CTRL_SMC             (1 << 3)
 271 #define MX51_ECSPI_CTRL_MODE_MASK       (0xf << 4)
 272 #define MX51_ECSPI_CTRL_DRCTL(drctl)    ((drctl) << 16)
 273 #define MX51_ECSPI_CTRL_POSTDIV_OFFSET  8
 274 #define MX51_ECSPI_CTRL_PREDIV_OFFSET   12
 275 #define MX51_ECSPI_CTRL_CS(cs)          ((cs & 3) << 18)
 276 #define MX51_ECSPI_CTRL_BL_OFFSET       20
 277 #define MX51_ECSPI_CTRL_BL_MASK         (0xfff << 20)
 278
 279 #define MX51_ECSPI_CONFIG       0x0c
 280 #define MX51_ECSPI_CONFIG_SCLKPHA(cs)   (1 << ((cs & 3) +  0))
 281 #define MX51_ECSPI_CONFIG_SCLKPOL(cs)   (1 << ((cs & 3) +  4))
 282 #define MX51_ECSPI_CONFIG_SBBCTRL(cs)   (1 << ((cs & 3) +  8))
 283 #define MX51_ECSPI_CONFIG_SSBPOL(cs)    (1 << ((cs & 3) + 12))
 284 #define MX51_ECSPI_CONFIG_SCLKCTL(cs)   (1 << ((cs & 3) + 20))
 285
 286 #define MX51_ECSPI_INT          0x10
 287 #define MX51_ECSPI_INT_TEEN             (1 <<  0)
 288 #define MX51_ECSPI_INT_RREN             (1 <<  3)
 289 #define MX51_ECSPI_INT_RDREN            (1 <<  4)
 290
 291 #define MX51_ECSPI_DMA          0x14
 292 #define MX51_ECSPI_DMA_TX_WML(wml)      ((wml) & 0x3f)
 293 #define MX51_ECSPI_DMA_RX_WML(wml)      (((wml) & 0x3f) << 16)
 294 #define MX51_ECSPI_DMA_RXT_WML(wml)     (((wml) & 0x3f) << 24)
 295
 296 #define MX51_ECSPI_DMA_TEDEN            (1 << 7)
 297 #define MX51_ECSPI_DMA_RXDEN            (1 << 23)
 298 #define MX51_ECSPI_DMA_RXTDEN           (1 << 31)
 299
 300 #define MX51_ECSPI_STAT         0x18
 301 #define MX51_ECSPI_STAT_RR              (1 <<  3)
 302
 303 #define MX51_ECSPI_TESTREG      0x20
 304 #define MX51_ECSPI_TESTREG_LBC  BIT(31)
 305
 306 static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
 307 {
 308         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);
 309
 310         if (spi_imx->rx_buf) {
 311 #ifdef __LITTLE_ENDIAN
 312                 unsigned int bytes_per_word;
 313
 314                 bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 315                 if (bytes_per_word == 1)
 316                         swab32s(&val);
 317                 else if (bytes_per_word == 2)
 318                         swahw32s(&val);
 319 #endif
 320                 *(u32 *)spi_imx->rx_buf = val;
 321                 spi_imx->rx_buf += sizeof(u32);
 322         }
 323
 324         spi_imx->remainder -= sizeof(u32);
 325 }
 326
 327 static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
 328 {
 329         int unaligned;
 330         u32 val;
 331
 332         unaligned = spi_imx->remainder % 4;
 333
 334         if (!unaligned) {
 335                 spi_imx_buf_rx_swap_u32(spi_imx);
 336                 return;
 337         }
 338
 339         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 340                 spi_imx_buf_rx_u16(spi_imx);
 341                 return;
 342         }
 343
 344         val = readl(spi_imx->base + MXC_CSPIRXDATA);
 345
 346         while (unaligned--) {
 347                 if (spi_imx->rx_buf) {
 348                         *(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;
 349                         spi_imx->rx_buf++;
 350                 }
 351                 spi_imx->remainder--;
 352         }
 353 }
 354
 355 static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
 356 {
 357         u32 val = 0;
 358 #ifdef __LITTLE_ENDIAN
 359         unsigned int bytes_per_word;
 360 #endif
 361
 362         if (spi_imx->tx_buf) {
 363                 val = *(u32 *)spi_imx->tx_buf;
 364                 spi_imx->tx_buf += sizeof(u32);
 365         }
 366
 367         spi_imx->count -= sizeof(u32);
 368 #ifdef __LITTLE_ENDIAN
 369         bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 370
 371         if (bytes_per_word == 1)
 372                 swab32s(&val);
 373         else if (bytes_per_word == 2)
 374                 swahw32s(&val);
 375 #endif
 376         writel(val, spi_imx->base + MXC_CSPITXDATA);
 377 }
 378
 379 static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
 380 {
 381         int unaligned;
 382         u32 val = 0;
 383
 384         unaligned = spi_imx->count % 4;
 385
 386         if (!unaligned) {
 387                 spi_imx_buf_tx_swap_u32(spi_imx);
 388                 return;
 389         }
 390
 391         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 392                 spi_imx_buf_tx_u16(spi_imx);
 393                 return;
 394         }
 395
 396         while (unaligned--) {
 397                 if (spi_imx->tx_buf) {
 398                         val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);
 399                         spi_imx->tx_buf++;
 400                 }
 401                 spi_imx->count--;
 402         }
 403
 404         writel(val, spi_imx->base + MXC_CSPITXDATA);
 405 }
 406
 407 static void mx53_ecspi_rx_slave(struct spi_imx_data *spi_imx)
 408 {
 409         u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));
 410
 411         if (spi_imx->rx_buf) {
 412                 int n_bytes = spi_imx->slave_burst % sizeof(val);
 413
 414                 if (!n_bytes)
 415                         n_bytes = sizeof(val);
 416
 417                 memcpy(spi_imx->rx_buf,
 418                        ((u8 *)&val) + sizeof(val) - n_bytes, n_bytes);
 419
 420                 spi_imx->rx_buf += n_bytes;
 421                 spi_imx->slave_burst -= n_bytes;
 422         }
 423
 424         spi_imx->remainder -= sizeof(u32);
 425 }
 426
 427 static void mx53_ecspi_tx_slave(struct spi_imx_data *spi_imx)
 428 {
 429         u32 val = 0;
 430         int n_bytes = spi_imx->count % sizeof(val);
 431
 432         if (!n_bytes)
 433                 n_bytes = sizeof(val);
 434
 435         if (spi_imx->tx_buf) {
 436                 memcpy(((u8 *)&val) + sizeof(val) - n_bytes,
 437                        spi_imx->tx_buf, n_bytes);
 438                 val = cpu_to_be32(val);
 439                 spi_imx->tx_buf += n_bytes;
 440         }
 441
 442         spi_imx->count -= n_bytes;
 443
 444         writel(val, spi_imx->base + MXC_CSPITXDATA);
 445 }
 446
 447 /* MX51 eCSPI */
 448 static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
 449                                       unsigned int fspi, unsigned int *fres)
 450 {
 451         /*
 452          * there are two 4-bit dividers, the pre-divider divides by
 453          * $pre, the post-divider by 2^$post
 454          */
 455         unsigned int pre, post;
 456         unsigned int fin = spi_imx->spi_clk;
 457
 458         fspi = min(fspi, fin);
 459
 460         post = fls(fin) - fls(fspi);
 461         if (fin > fspi << post)
 462                 post++;
 463
 464         /* now we have: (fin <= fspi << post) with post being minimal */
 465
 466         post = max(4U, post) - 4;
 467         if (unlikely(post > 0xf)) {
 468                 dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",
 469                                 fspi, fin);
 470                 return 0xff;
 471         }
 472
 473         pre = DIV_ROUND_UP(fin, fspi << post) - 1;
 474
 475         dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",
 476                         __func__, fin, fspi, post, pre);
 477
 478         /* Resulting frequency for the SCLK line. */
 479         *fres = (fin / (pre + 1)) >> post;
 480
 481         return (pre << MX51_ECSPI_CTRL_PREDIV_OFFSET) |
 482                 (post << MX51_ECSPI_CTRL_POSTDIV_OFFSET);
 483 }
 484
 485 static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)
 486 {
 487         unsigned int val = 0;
 488
 489         if (enable & MXC_INT_TE)
 490                 val |= MX51_ECSPI_INT_TEEN;
 491
 492         if (enable & MXC_INT_RR)
 493                 val |= MX51_ECSPI_INT_RREN;
 494
 495         if (enable & MXC_INT_RDR)
 496                 val |= MX51_ECSPI_INT_RDREN;
 497
 498         writel(val, spi_imx->base + MX51_ECSPI_INT);
 499 }
 500
 501 static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)
 502 {
 503         u32 reg;
 504
 505         reg = readl(spi_imx->base + MX51_ECSPI_CTRL);
 506         reg |= MX51_ECSPI_CTRL_XCH;
 507         writel(reg, spi_imx->base + MX51_ECSPI_CTRL);
 508 }
 509
 510 static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)
 511 {
 512         u32 ctrl;
 513
 514         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 515         ctrl &= ~MX51_ECSPI_CTRL_ENABLE;
 516         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 517 }
 518
 519 static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 520                                       struct spi_message *msg)
 521 {
 522         struct spi_device *spi = msg->spi;
 523         struct spi_transfer *xfer;
 524         u32 ctrl = MX51_ECSPI_CTRL_ENABLE;
 525         u32 min_speed_hz = ~0U;
 526         u32 testreg, delay;
 527         u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 528         u32 current_cfg = cfg;
 529
 530         /* set Master or Slave mode */
 531         if (spi_imx->slave_mode)
 532                 ctrl &= ~MX51_ECSPI_CTRL_MODE_MASK;
 533         else
 534                 ctrl |= MX51_ECSPI_CTRL_MODE_MASK;
 535
 536         /*
 537          * Enable SPI_RDY handling (falling edge/level triggered).
 538          */
 539         if (spi->mode & SPI_READY)
 540                 ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
 541
 542         /* set chip select to use */
 543         ctrl |= MX51_ECSPI_CTRL_CS(spi_get_chipselect(spi, 0));
 544
 545         /*
 546          * The ctrl register must be written first, with the EN bit set other
 547          * registers must not be written to.
 548          */
 549         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 550
 551         testreg = readl(spi_imx->base + MX51_ECSPI_TESTREG);
 552         if (spi->mode & SPI_LOOP)
 553                 testreg |= MX51_ECSPI_TESTREG_LBC;
 554         else
 555                 testreg &= ~MX51_ECSPI_TESTREG_LBC;
 556         writel(testreg, spi_imx->base + MX51_ECSPI_TESTREG);
 557
 558         /*
 559          * eCSPI burst completion by Chip Select signal in Slave mode
 560          * is not functional for imx53 Soc, config SPI burst completed when
 561          * BURST_LENGTH + 1 bits are received
 562          */
 563         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 564                 cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(spi_get_chipselect(spi, 0));
 565         else
 566                 cfg |= MX51_ECSPI_CONFIG_SBBCTRL(spi_get_chipselect(spi, 0));
 567
 568         if (spi->mode & SPI_CPOL) {
 569                 cfg |= MX51_ECSPI_CONFIG_SCLKPOL(spi_get_chipselect(spi, 0));
 570                 cfg |= MX51_ECSPI_CONFIG_SCLKCTL(spi_get_chipselect(spi, 0));
 571         } else {
 572                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(spi_get_chipselect(spi, 0));
 573                 cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(spi_get_chipselect(spi, 0));
 574         }
 575
 576         if (spi->mode & SPI_CS_HIGH)
 577                 cfg |= MX51_ECSPI_CONFIG_SSBPOL(spi_get_chipselect(spi, 0));
 578         else
 579                 cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(spi_get_chipselect(spi, 0));
 580
 581         if (cfg == current_cfg)
 582                 return 0;
 583
 584         writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 585
 586         /*
 587          * Wait until the changes in the configuration register CONFIGREG
 588          * propagate into the hardware. It takes exactly one tick of the
 589          * SCLK clock, but we will wait two SCLK clock just to be sure. The
 590          * effect of the delay it takes for the hardware to apply changes
 591          * is noticable if the SCLK clock run very slow. In such a case, if
 592          * the polarity of SCLK should be inverted, the GPIO ChipSelect might
 593          * be asserted before the SCLK polarity changes, which would disrupt
 594          * the SPI communication as the device on the other end would consider
 595          * the change of SCLK polarity as a clock tick already.
 596          *
 597          * Because spi_imx->spi_bus_clk is only set in prepare_message
 598          * callback, iterate over all the transfers in spi_message, find the
 599          * one with lowest bus frequency, and use that bus frequency for the
 600          * delay calculation. In case all transfers have speed_hz == 0, then
 601          * min_speed_hz is ~0 and the resulting delay is zero.
 602          */
 603         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 604                 if (!xfer->speed_hz)
 605                         continue;
 606                 min_speed_hz = min(xfer->speed_hz, min_speed_hz);
 607         }
 608
 609         delay = (2 * 1000000) / min_speed_hz;
 610         if (likely(delay < 10)) /* SCLK is faster than 200 kHz */
 611                 udelay(delay);
 612         else                    /* SCLK is _very_ slow */
 613                 usleep_range(delay, delay + 10);
 614
 615         return 0;
 616 }
 617
 618 static void mx51_configure_cpha(struct spi_imx_data *spi_imx,
 619                                 struct spi_device *spi)
 620 {
 621         bool cpha = (spi->mode & SPI_CPHA);
 622         bool flip_cpha = (spi->mode & SPI_RX_CPHA_FLIP) && spi_imx->rx_only;
 623         u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 624
 625         /* Flip cpha logical value iff flip_cpha */
 626         cpha ^= flip_cpha;
 627
 628         if (cpha)
 629                 cfg |= MX51_ECSPI_CONFIG_SCLKPHA(spi_get_chipselect(spi, 0));
 630         else
 631                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(spi_get_chipselect(spi, 0));
 632
 633         writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 634 }
 635
 636 static int mx51_ecspi_prepare_transfer(struct spi_imx_data *spi_imx,
 637                                        struct spi_device *spi)
 638 {
 639         u32 ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 640         u32 clk;
 641
 642         /* Clear BL field and set the right value */
 643         ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
 644         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 645                 ctrl |= (spi_imx->slave_burst * 8 - 1)
 646                         << MX51_ECSPI_CTRL_BL_OFFSET;
 647         else
 648                 ctrl |= (spi_imx->bits_per_word - 1)
 649                         << MX51_ECSPI_CTRL_BL_OFFSET;
 650
 651         /* set clock speed */
 652         ctrl &= ~(0xf << MX51_ECSPI_CTRL_POSTDIV_OFFSET |
 653                   0xf << MX51_ECSPI_CTRL_PREDIV_OFFSET);
 654         ctrl |= mx51_ecspi_clkdiv(spi_imx, spi_imx->spi_bus_clk, &clk);
 655         spi_imx->spi_bus_clk = clk;
 656
 657         mx51_configure_cpha(spi_imx, spi);
 658
 659         /*
 660          * ERR009165: work in XHC mode instead of SMC as PIO on the chips
 661          * before i.mx6ul.
 662          */
 663         if (spi_imx->usedma && spi_imx->devtype_data->tx_glitch_fixed)
 664                 ctrl |= MX51_ECSPI_CTRL_SMC;
 665         else
 666                 ctrl &= ~MX51_ECSPI_CTRL_SMC;
 667
 668         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 669
 670         return 0;
 671 }
 672
 673 static void mx51_setup_wml(struct spi_imx_data *spi_imx)
 674 {
 675         u32 tx_wml = 0;
 676
 677         if (spi_imx->devtype_data->tx_glitch_fixed)
 678                 tx_wml = spi_imx->wml;
 679         /*
 680          * Configure the DMA register: setup the watermark
 681          * and enable DMA request.
 682          */
 683         writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) |
 684                 MX51_ECSPI_DMA_TX_WML(tx_wml) |
 685                 MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |
 686                 MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN |
 687                 MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA);
 688 }
 689
 690 static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)
 691 {
 692         return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;
 693 }
 694
 695 static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 696 {
 697         /* drain receive buffer */
 698         while (mx51_ecspi_rx_available(spi_imx))
 699                 readl(spi_imx->base + MXC_CSPIRXDATA);
 700 }
 701
 702 #define MX31_INTREG_TEEN        (1 << 0)
 703 #define MX31_INTREG_RREN        (1 << 3)
 704
 705 #define MX31_CSPICTRL_ENABLE    (1 << 0)
 706 #define MX31_CSPICTRL_MASTER    (1 << 1)
 707 #define MX31_CSPICTRL_XCH       (1 << 2)
 708 #define MX31_CSPICTRL_SMC       (1 << 3)
 709 #define MX31_CSPICTRL_POL       (1 << 4)
 710 #define MX31_CSPICTRL_PHA       (1 << 5)
 711 #define MX31_CSPICTRL_SSCTL     (1 << 6)
 712 #define MX31_CSPICTRL_SSPOL     (1 << 7)
 713 #define MX31_CSPICTRL_BC_SHIFT  8
 714 #define MX35_CSPICTRL_BL_SHIFT  20
 715 #define MX31_CSPICTRL_CS_SHIFT  24
 716 #define MX35_CSPICTRL_CS_SHIFT  12
 717 #define MX31_CSPICTRL_DR_SHIFT  16
 718
 719 #define MX31_CSPI_DMAREG        0x10
 720 #define MX31_DMAREG_RH_DEN      (1<<4)
 721 #define MX31_DMAREG_TH_DEN      (1<<1)
 722
 723 #define MX31_CSPISTATUS         0x14
 724 #define MX31_STATUS_RR          (1 << 3)
 725
 726 #define MX31_CSPI_TESTREG       0x1C
 727 #define MX31_TEST_LBC           (1 << 14)
 728
 729 /* These functions also work for the i.MX35, but be aware that
 730  * the i.MX35 has a slightly different register layout for bits
 731  * we do not use here.
 732  */
 733 static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)
 734 {
 735         unsigned int val = 0;
 736
 737         if (enable & MXC_INT_TE)
 738                 val |= MX31_INTREG_TEEN;
 739         if (enable & MXC_INT_RR)
 740                 val |= MX31_INTREG_RREN;
 741
 742         writel(val, spi_imx->base + MXC_CSPIINT);
 743 }
 744
 745 static void mx31_trigger(struct spi_imx_data *spi_imx)
 746 {
 747         unsigned int reg;
 748
 749         reg = readl(spi_imx->base + MXC_CSPICTRL);
 750         reg |= MX31_CSPICTRL_XCH;
 751         writel(reg, spi_imx->base + MXC_CSPICTRL);
 752 }
 753
 754 static int mx31_prepare_message(struct spi_imx_data *spi_imx,
 755                                 struct spi_message *msg)
 756 {
 757         return 0;
 758 }
 759
 760 static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,
 761                                  struct spi_device *spi)
 762 {
 763         unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_MASTER;
 764         unsigned int clk;
 765
 766         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 767                 MX31_CSPICTRL_DR_SHIFT;
 768         spi_imx->spi_bus_clk = clk;
 769
 770         if (is_imx35_cspi(spi_imx)) {
 771                 reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;
 772                 reg |= MX31_CSPICTRL_SSCTL;
 773         } else {
 774                 reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;
 775         }
 776
 777         if (spi->mode & SPI_CPHA)
 778                 reg |= MX31_CSPICTRL_PHA;
 779         if (spi->mode & SPI_CPOL)
 780                 reg |= MX31_CSPICTRL_POL;
 781         if (spi->mode & SPI_CS_HIGH)
 782                 reg |= MX31_CSPICTRL_SSPOL;
 783         if (!spi_get_csgpiod(spi, 0))
 784                 reg |= (spi_get_chipselect(spi, 0)) <<
 785                         (is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 786                                                   MX31_CSPICTRL_CS_SHIFT);
 787
 788         if (spi_imx->usedma)
 789                 reg |= MX31_CSPICTRL_SMC;
 790
 791         writel(reg, spi_imx->base + MXC_CSPICTRL);
 792
 793         reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
 794         if (spi->mode & SPI_LOOP)
 795                 reg |= MX31_TEST_LBC;
 796         else
 797                 reg &= ~MX31_TEST_LBC;
 798         writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
 799
 800         if (spi_imx->usedma) {
 801                 /*
 802                  * configure DMA requests when RXFIFO is half full and
 803                  * when TXFIFO is half empty
 804                  */
 805                 writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
 806                         spi_imx->base + MX31_CSPI_DMAREG);
 807         }
 808
 809         return 0;
 810 }
 811
 812 static int mx31_rx_available(struct spi_imx_data *spi_imx)
 813 {
 814         return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;
 815 }
 816
 817 static void mx31_reset(struct spi_imx_data *spi_imx)
 818 {
 819         /* drain receive buffer */
 820         while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)
 821                 readl(spi_imx->base + MXC_CSPIRXDATA);
 822 }
 823
 824 #define MX21_INTREG_RR          (1 << 4)
 825 #define MX21_INTREG_TEEN        (1 << 9)
 826 #define MX21_INTREG_RREN        (1 << 13)
 827
 828 #define MX21_CSPICTRL_POL       (1 << 5)
 829 #define MX21_CSPICTRL_PHA       (1 << 6)
 830 #define MX21_CSPICTRL_SSPOL     (1 << 8)
 831 #define MX21_CSPICTRL_XCH       (1 << 9)
 832 #define MX21_CSPICTRL_ENABLE    (1 << 10)
 833 #define MX21_CSPICTRL_MASTER    (1 << 11)
 834 #define MX21_CSPICTRL_DR_SHIFT  14
 835 #define MX21_CSPICTRL_CS_SHIFT  19
 836
 837 static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)
 838 {
 839         unsigned int val = 0;
 840
 841         if (enable & MXC_INT_TE)
 842                 val |= MX21_INTREG_TEEN;
 843         if (enable & MXC_INT_RR)
 844                 val |= MX21_INTREG_RREN;
 845
 846         writel(val, spi_imx->base + MXC_CSPIINT);
 847 }
 848
 849 static void mx21_trigger(struct spi_imx_data *spi_imx)
 850 {
 851         unsigned int reg;
 852
 853         reg = readl(spi_imx->base + MXC_CSPICTRL);
 854         reg |= MX21_CSPICTRL_XCH;
 855         writel(reg, spi_imx->base + MXC_CSPICTRL);
 856 }
 857
 858 static int mx21_prepare_message(struct spi_imx_data *spi_imx,
 859                                 struct spi_message *msg)
 860 {
 861         return 0;
 862 }
 863
 864 static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,
 865                                  struct spi_device *spi)
 866 {
 867         unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_MASTER;
 868         unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;
 869         unsigned int clk;
 870
 871         reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, spi_imx->spi_bus_clk, max, &clk)
 872                 << MX21_CSPICTRL_DR_SHIFT;
 873         spi_imx->spi_bus_clk = clk;
 874
 875         reg |= spi_imx->bits_per_word - 1;
 876
 877         if (spi->mode & SPI_CPHA)
 878                 reg |= MX21_CSPICTRL_PHA;
 879         if (spi->mode & SPI_CPOL)
 880                 reg |= MX21_CSPICTRL_POL;
 881         if (spi->mode & SPI_CS_HIGH)
 882                 reg |= MX21_CSPICTRL_SSPOL;
 883         if (!spi_get_csgpiod(spi, 0))
 884                 reg |= spi_get_chipselect(spi, 0) << MX21_CSPICTRL_CS_SHIFT;
 885
 886         writel(reg, spi_imx->base + MXC_CSPICTRL);
 887
 888         return 0;
 889 }
 890
 891 static int mx21_rx_available(struct spi_imx_data *spi_imx)
 892 {
 893         return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;
 894 }
 895
 896 static void mx21_reset(struct spi_imx_data *spi_imx)
 897 {
 898         writel(1, spi_imx->base + MXC_RESET);
 899 }
 900
 901 #define MX1_INTREG_RR           (1 << 3)
 902 #define MX1_INTREG_TEEN         (1 << 8)
 903 #define MX1_INTREG_RREN         (1 << 11)
 904
 905 #define MX1_CSPICTRL_POL        (1 << 4)
 906 #define MX1_CSPICTRL_PHA        (1 << 5)
 907 #define MX1_CSPICTRL_XCH        (1 << 8)
 908 #define MX1_CSPICTRL_ENABLE     (1 << 9)
 909 #define MX1_CSPICTRL_MASTER     (1 << 10)
 910 #define MX1_CSPICTRL_DR_SHIFT   13
 911
 912 static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)
 913 {
 914         unsigned int val = 0;
 915
 916         if (enable & MXC_INT_TE)
 917                 val |= MX1_INTREG_TEEN;
 918         if (enable & MXC_INT_RR)
 919                 val |= MX1_INTREG_RREN;
 920
 921         writel(val, spi_imx->base + MXC_CSPIINT);
 922 }
 923
 924 static void mx1_trigger(struct spi_imx_data *spi_imx)
 925 {
 926         unsigned int reg;
 927
 928         reg = readl(spi_imx->base + MXC_CSPICTRL);
 929         reg |= MX1_CSPICTRL_XCH;
 930         writel(reg, spi_imx->base + MXC_CSPICTRL);
 931 }
 932
 933 static int mx1_prepare_message(struct spi_imx_data *spi_imx,
 934                                struct spi_message *msg)
 935 {
 936         return 0;
 937 }
 938
 939 static int mx1_prepare_transfer(struct spi_imx_data *spi_imx,
 940                                 struct spi_device *spi)
 941 {
 942         unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_MASTER;
 943         unsigned int clk;
 944
 945         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 946                 MX1_CSPICTRL_DR_SHIFT;
 947         spi_imx->spi_bus_clk = clk;
 948
 949         reg |= spi_imx->bits_per_word - 1;
 950
 951         if (spi->mode & SPI_CPHA)
 952                 reg |= MX1_CSPICTRL_PHA;
 953         if (spi->mode & SPI_CPOL)
 954                 reg |= MX1_CSPICTRL_POL;
 955
 956         writel(reg, spi_imx->base + MXC_CSPICTRL);
 957
 958         return 0;
 959 }
 960
 961 static int mx1_rx_available(struct spi_imx_data *spi_imx)
 962 {
 963         return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;
 964 }
 965
 966 static void mx1_reset(struct spi_imx_data *spi_imx)
 967 {
 968         writel(1, spi_imx->base + MXC_RESET);
 969 }
 970
 971 static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
 972         .intctrl = mx1_intctrl,
 973         .prepare_message = mx1_prepare_message,
 974         .prepare_transfer = mx1_prepare_transfer,
 975         .trigger = mx1_trigger,
 976         .rx_available = mx1_rx_available,
 977         .reset = mx1_reset,
 978         .fifo_size = 8,
 979         .has_dmamode = false,
 980         .dynamic_burst = false,
 981         .has_slavemode = false,
 982         .devtype = IMX1_CSPI,
 983 };
 984
 985 static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
 986         .intctrl = mx21_intctrl,
 987         .prepare_message = mx21_prepare_message,
 988         .prepare_transfer = mx21_prepare_transfer,
 989         .trigger = mx21_trigger,
 990         .rx_available = mx21_rx_available,
 991         .reset = mx21_reset,
 992         .fifo_size = 8,
 993         .has_dmamode = false,
 994         .dynamic_burst = false,
 995         .has_slavemode = false,
 996         .devtype = IMX21_CSPI,
 997 };
 998
 999 static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
1000         /* i.mx27 cspi shares the functions with i.mx21 one */
1001         .intctrl = mx21_intctrl,
1002         .prepare_message = mx21_prepare_message,
1003         .prepare_transfer = mx21_prepare_transfer,
1004         .trigger = mx21_trigger,
1005         .rx_available = mx21_rx_available,
1006         .reset = mx21_reset,
1007         .fifo_size = 8,
1008         .has_dmamode = false,
1009         .dynamic_burst = false,
1010         .has_slavemode = false,
1011         .devtype = IMX27_CSPI,
1012 };
1013
1014 static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
1015         .intctrl = mx31_intctrl,
1016         .prepare_message = mx31_prepare_message,
1017         .prepare_transfer = mx31_prepare_transfer,
1018         .trigger = mx31_trigger,
1019         .rx_available = mx31_rx_available,
1020         .reset = mx31_reset,
1021         .fifo_size = 8,
1022         .has_dmamode = false,
1023         .dynamic_burst = false,
1024         .has_slavemode = false,
1025         .devtype = IMX31_CSPI,
1026 };
1027
1028 static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
1029         /* i.mx35 and later cspi shares the functions with i.mx31 one */
1030         .intctrl = mx31_intctrl,
1031         .prepare_message = mx31_prepare_message,
1032         .prepare_transfer = mx31_prepare_transfer,
1033         .trigger = mx31_trigger,
1034         .rx_available = mx31_rx_available,
1035         .reset = mx31_reset,
1036         .fifo_size = 8,
1037         .has_dmamode = true,
1038         .dynamic_burst = false,
1039         .has_slavemode = false,
1040         .devtype = IMX35_CSPI,
1041 };
1042
1043 static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
1044         .intctrl = mx51_ecspi_intctrl,
1045         .prepare_message = mx51_ecspi_prepare_message,
1046         .prepare_transfer = mx51_ecspi_prepare_transfer,
1047         .trigger = mx51_ecspi_trigger,
1048         .rx_available = mx51_ecspi_rx_available,
1049         .reset = mx51_ecspi_reset,
1050         .setup_wml = mx51_setup_wml,
1051         .fifo_size = 64,
1052         .has_dmamode = true,
1053         .dynamic_burst = true,
1054         .has_slavemode = true,
1055         .disable = mx51_ecspi_disable,
1056         .devtype = IMX51_ECSPI,
1057 };
1058
1059 static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
1060         .intctrl = mx51_ecspi_intctrl,
1061         .prepare_message = mx51_ecspi_prepare_message,
1062         .prepare_transfer = mx51_ecspi_prepare_transfer,
1063         .trigger = mx51_ecspi_trigger,
1064         .rx_available = mx51_ecspi_rx_available,
1065         .reset = mx51_ecspi_reset,
1066         .fifo_size = 64,
1067         .has_dmamode = true,
1068         .has_slavemode = true,
1069         .disable = mx51_ecspi_disable,
1070         .devtype = IMX53_ECSPI,
1071 };
1072
1073 static struct spi_imx_devtype_data imx6ul_ecspi_devtype_data = {
1074         .intctrl = mx51_ecspi_intctrl,
1075         .prepare_message = mx51_ecspi_prepare_message,
1076         .prepare_transfer = mx51_ecspi_prepare_transfer,
1077         .trigger = mx51_ecspi_trigger,
1078         .rx_available = mx51_ecspi_rx_available,
1079         .reset = mx51_ecspi_reset,
1080         .setup_wml = mx51_setup_wml,
1081         .fifo_size = 64,
1082         .has_dmamode = true,
1083         .dynamic_burst = true,
1084         .has_slavemode = true,
1085         .tx_glitch_fixed = true,
1086         .disable = mx51_ecspi_disable,
1087         .devtype = IMX51_ECSPI,
1088 };
1089
1090 static const struct of_device_id spi_imx_dt_ids[] = {
1091         { .compatible = "fsl,imx1-cspi", .data = &imx1_cspi_devtype_data, },
1092         { .compatible = "fsl,imx21-cspi", .data = &imx21_cspi_devtype_data, },
1093         { .compatible = "fsl,imx27-cspi", .data = &imx27_cspi_devtype_data, },
1094         { .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
1095         { .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
1096         { .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
1097         { .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
1098         { .compatible = "fsl,imx6ul-ecspi", .data = &imx6ul_ecspi_devtype_data, },
1099         { /* sentinel */ }
1100 };
1101 MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
1102
1103 static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)
1104 {
1105         u32 ctrl;
1106
1107         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
1108         ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
1109         ctrl |= ((n_bits - 1) << MX51_ECSPI_CTRL_BL_OFFSET);
1110         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
1111 }
1112
1113 static void spi_imx_push(struct spi_imx_data *spi_imx)
1114 {
1115         unsigned int burst_len;
1116
1117         /*
1118          * Reload the FIFO when the remaining bytes to be transferred in the
1119          * current burst is 0. This only applies when bits_per_word is a
1120          * multiple of 8.
1121          */
1122         if (!spi_imx->remainder) {
1123                 if (spi_imx->dynamic_burst) {
1124
1125                         /* We need to deal unaligned data first */
1126                         burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;
1127
1128                         if (!burst_len)
1129                                 burst_len = MX51_ECSPI_CTRL_MAX_BURST;
1130
1131                         spi_imx_set_burst_len(spi_imx, burst_len * 8);
1132
1133                         spi_imx->remainder = burst_len;
1134                 } else {
1135                         spi_imx->remainder = spi_imx_bytes_per_word(spi_imx->bits_per_word);
1136                 }
1137         }
1138
1139         while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
1140                 if (!spi_imx->count)
1141                         break;
1142                 if (spi_imx->dynamic_burst &&
1143                     spi_imx->txfifo >= DIV_ROUND_UP(spi_imx->remainder, 4))
1144                         break;
1145                 spi_imx->tx(spi_imx);
1146                 spi_imx->txfifo++;
1147         }
1148
1149         if (!spi_imx->slave_mode)
1150                 spi_imx->devtype_data->trigger(spi_imx);
1151 }
1152
1153 static irqreturn_t spi_imx_isr(int irq, void *dev_id)
1154 {
1155         struct spi_imx_data *spi_imx = dev_id;
1156
1157         while (spi_imx->txfifo &&
1158                spi_imx->devtype_data->rx_available(spi_imx)) {
1159                 spi_imx->rx(spi_imx);
1160                 spi_imx->txfifo--;
1161         }
1162
1163         if (spi_imx->count) {
1164                 spi_imx_push(spi_imx);
1165                 return IRQ_HANDLED;
1166         }
1167
1168         if (spi_imx->txfifo) {
1169                 /* No data left to push, but still waiting for rx data,
1170                  * enable receive data available interrupt.
1171                  */
1172                 spi_imx->devtype_data->intctrl(
1173                                 spi_imx, MXC_INT_RR);
1174                 return IRQ_HANDLED;
1175         }
1176
1177         spi_imx->devtype_data->intctrl(spi_imx, 0);
1178         complete(&spi_imx->xfer_done);
1179
1180         return IRQ_HANDLED;
1181 }
1182
1183 static int spi_imx_dma_configure(struct spi_controller *controller)
1184 {
1185         int ret;
1186         enum dma_slave_buswidth buswidth;
1187         struct dma_slave_config rx = {}, tx = {};
1188         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1189
1190         switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
1191         case 4:
1192                 buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
1193                 break;
1194         case 2:
1195                 buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
1196                 break;
1197         case 1:
1198                 buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
1199                 break;
1200         default:
1201                 return -EINVAL;
1202         }
1203
1204         tx.direction = DMA_MEM_TO_DEV;
1205         tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
1206         tx.dst_addr_width = buswidth;
1207         tx.dst_maxburst = spi_imx->wml;
1208         ret = dmaengine_slave_config(controller->dma_tx, &tx);
1209         if (ret) {
1210                 dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
1211                 return ret;
1212         }
1213
1214         rx.direction = DMA_DEV_TO_MEM;
1215         rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
1216         rx.src_addr_width = buswidth;
1217         rx.src_maxburst = spi_imx->wml;
1218         ret = dmaengine_slave_config(controller->dma_rx, &rx);
1219         if (ret) {
1220                 dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
1221                 return ret;
1222         }
1223
1224         return 0;
1225 }
1226
1227 static int spi_imx_setupxfer(struct spi_device *spi,
1228                                  struct spi_transfer *t)
1229 {
1230         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1231
1232         if (!t)
1233                 return 0;
1234
1235         if (!t->speed_hz) {
1236                 if (!spi->max_speed_hz) {
1237                         dev_err(&spi->dev, "no speed_hz provided!\n");
1238                         return -EINVAL;
1239                 }
1240                 dev_dbg(&spi->dev, "using spi->max_speed_hz!\n");
1241                 spi_imx->spi_bus_clk = spi->max_speed_hz;
1242         } else
1243                 spi_imx->spi_bus_clk = t->speed_hz;
1244
1245         spi_imx->bits_per_word = t->bits_per_word;
1246
1247         /*
1248          * Initialize the functions for transfer. To transfer non byte-aligned
1249          * words, we have to use multiple word-size bursts, we can't use
1250          * dynamic_burst in that case.
1251          */
1252         if (spi_imx->devtype_data->dynamic_burst && !spi_imx->slave_mode &&
1253             !(spi->mode & SPI_CS_WORD) &&
1254             (spi_imx->bits_per_word == 8 ||
1255             spi_imx->bits_per_word == 16 ||
1256             spi_imx->bits_per_word == 32)) {
1257
1258                 spi_imx->rx = spi_imx_buf_rx_swap;
1259                 spi_imx->tx = spi_imx_buf_tx_swap;
1260                 spi_imx->dynamic_burst = 1;
1261
1262         } else {
1263                 if (spi_imx->bits_per_word <= 8) {
1264                         spi_imx->rx = spi_imx_buf_rx_u8;
1265                         spi_imx->tx = spi_imx_buf_tx_u8;
1266                 } else if (spi_imx->bits_per_word <= 16) {
1267                         spi_imx->rx = spi_imx_buf_rx_u16;
1268                         spi_imx->tx = spi_imx_buf_tx_u16;
1269                 } else {
1270                         spi_imx->rx = spi_imx_buf_rx_u32;
1271                         spi_imx->tx = spi_imx_buf_tx_u32;
1272                 }
1273                 spi_imx->dynamic_burst = 0;
1274         }
1275
1276         if (spi_imx_can_dma(spi_imx->controller, spi, t))
1277                 spi_imx->usedma = true;
1278         else
1279                 spi_imx->usedma = false;
1280
1281         spi_imx->rx_only = ((t->tx_buf == NULL)
1282                         || (t->tx_buf == spi->controller->dummy_tx));
1283
1284         if (is_imx53_ecspi(spi_imx) && spi_imx->slave_mode) {
1285                 spi_imx->rx = mx53_ecspi_rx_slave;
1286                 spi_imx->tx = mx53_ecspi_tx_slave;
1287                 spi_imx->slave_burst = t->len;
1288         }
1289
1290         spi_imx->devtype_data->prepare_transfer(spi_imx, spi);
1291
1292         return 0;
1293 }
1294
1295 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
1296 {
1297         struct spi_controller *controller = spi_imx->controller;
1298
1299         if (controller->dma_rx) {
1300                 dma_release_channel(controller->dma_rx);
1301                 controller->dma_rx = NULL;
1302         }
1303
1304         if (controller->dma_tx) {
1305                 dma_release_channel(controller->dma_tx);
1306                 controller->dma_tx = NULL;
1307         }
1308 }
1309
1310 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
1311                              struct spi_controller *controller)
1312 {
1313         int ret;
1314
1315         spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
1316
1317         /* Prepare for TX DMA: */
1318         controller->dma_tx = dma_request_chan(dev, "tx");
1319         if (IS_ERR(controller->dma_tx)) {
1320                 ret = PTR_ERR(controller->dma_tx);
1321                 dev_dbg(dev, "can't get the TX DMA channel, error %d!\n", ret);
1322                 controller->dma_tx = NULL;
1323                 goto err;
1324         }
1325
1326         /* Prepare for RX : */
1327         controller->dma_rx = dma_request_chan(dev, "rx");
1328         if (IS_ERR(controller->dma_rx)) {
1329                 ret = PTR_ERR(controller->dma_rx);
1330                 dev_dbg(dev, "can't get the RX DMA channel, error %d\n", ret);
1331                 controller->dma_rx = NULL;
1332                 goto err;
1333         }
1334
1335         init_completion(&spi_imx->dma_rx_completion);
1336         init_completion(&spi_imx->dma_tx_completion);
1337         controller->can_dma = spi_imx_can_dma;
1338         controller->max_dma_len = MAX_SDMA_BD_BYTES;
1339         spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
1340                                          SPI_CONTROLLER_MUST_TX;
1341
1342         return 0;
1343 err:
1344         spi_imx_sdma_exit(spi_imx);
1345         return ret;
1346 }
1347
1348 static void spi_imx_dma_rx_callback(void *cookie)
1349 {
1350         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1351
1352         complete(&spi_imx->dma_rx_completion);
1353 }
1354
1355 static void spi_imx_dma_tx_callback(void *cookie)
1356 {
1357         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1358
1359         complete(&spi_imx->dma_tx_completion);
1360 }
1361
1362 static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
1363 {
1364         unsigned long timeout = 0;
1365
1366         /* Time with actual data transfer and CS change delay related to HW */
1367         timeout = (8 + 4) * size / spi_imx->spi_bus_clk;
1368
1369         /* Add extra second for scheduler related activities */
1370         timeout += 1;
1371
1372         /* Double calculated timeout */
1373         return msecs_to_jiffies(2 * timeout * MSEC_PER_SEC);
1374 }
1375
1376 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
1377                                 struct spi_transfer *transfer)
1378 {
1379         struct dma_async_tx_descriptor *desc_tx, *desc_rx;
1380         unsigned long transfer_timeout;
1381         unsigned long timeout;
1382         struct spi_controller *controller = spi_imx->controller;
1383         struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
1384         struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);
1385         unsigned int bytes_per_word, i;
1386         int ret;
1387
1388         /* Get the right burst length from the last sg to ensure no tail data */
1389         bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
1390         for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
1391                 if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
1392                         break;
1393         }
1394         /* Use 1 as wml in case no available burst length got */
1395         if (i == 0)
1396                 i = 1;
1397
1398         spi_imx->wml =  i;
1399
1400         ret = spi_imx_dma_configure(controller);
1401         if (ret)
1402                 goto dma_failure_no_start;
1403
1404         if (!spi_imx->devtype_data->setup_wml) {
1405                 dev_err(spi_imx->dev, "No setup_wml()?\n");
1406                 ret = -EINVAL;
1407                 goto dma_failure_no_start;
1408         }
1409         spi_imx->devtype_data->setup_wml(spi_imx);
1410
1411         /*
1412          * The TX DMA setup starts the transfer, so make sure RX is configured
1413          * before TX.
1414          */
1415         desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,
1416                                 rx->sgl, rx->nents, DMA_DEV_TO_MEM,
1417                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1418         if (!desc_rx) {
1419                 ret = -EINVAL;
1420                 goto dma_failure_no_start;
1421         }
1422
1423         desc_rx->callback = spi_imx_dma_rx_callback;
1424         desc_rx->callback_param = (void *)spi_imx;
1425         dmaengine_submit(desc_rx);
1426         reinit_completion(&spi_imx->dma_rx_completion);
1427         dma_async_issue_pending(controller->dma_rx);
1428
1429         desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,
1430                                 tx->sgl, tx->nents, DMA_MEM_TO_DEV,
1431                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1432         if (!desc_tx) {
1433                 dmaengine_terminate_all(controller->dma_tx);
1434                 dmaengine_terminate_all(controller->dma_rx);
1435                 return -EINVAL;
1436         }
1437
1438         desc_tx->callback = spi_imx_dma_tx_callback;
1439         desc_tx->callback_param = (void *)spi_imx;
1440         dmaengine_submit(desc_tx);
1441         reinit_completion(&spi_imx->dma_tx_completion);
1442         dma_async_issue_pending(controller->dma_tx);
1443
1444         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1445
1446         /* Wait SDMA to finish the data transfer.*/
1447         timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
1448                                                 transfer_timeout);
1449         if (!timeout) {
1450                 dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
1451                 dmaengine_terminate_all(controller->dma_tx);
1452                 dmaengine_terminate_all(controller->dma_rx);
1453                 return -ETIMEDOUT;
1454         }
1455
1456         timeout = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
1457                                               transfer_timeout);
1458         if (!timeout) {
1459                 dev_err(&controller->dev, "I/O Error in DMA RX\n");
1460                 spi_imx->devtype_data->reset(spi_imx);
1461                 dmaengine_terminate_all(controller->dma_rx);
1462                 return -ETIMEDOUT;
1463         }
1464
1465         return 0;
1466 /* fallback to pio */
1467 dma_failure_no_start:
1468         transfer->error |= SPI_TRANS_FAIL_NO_START;
1469         return ret;
1470 }
1471
1472 static int spi_imx_pio_transfer(struct spi_device *spi,
1473                                 struct spi_transfer *transfer)
1474 {
1475         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1476         unsigned long transfer_timeout;
1477         unsigned long timeout;
1478
1479         spi_imx->tx_buf = transfer->tx_buf;
1480         spi_imx->rx_buf = transfer->rx_buf;
1481         spi_imx->count = transfer->len;
1482         spi_imx->txfifo = 0;
1483         spi_imx->remainder = 0;
1484
1485         reinit_completion(&spi_imx->xfer_done);
1486
1487         spi_imx_push(spi_imx);
1488
1489         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);
1490
1491         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1492
1493         timeout = wait_for_completion_timeout(&spi_imx->xfer_done,
1494                                               transfer_timeout);
1495         if (!timeout) {
1496                 dev_err(&spi->dev, "I/O Error in PIO\n");
1497                 spi_imx->devtype_data->reset(spi_imx);
1498                 return -ETIMEDOUT;
1499         }
1500
1501         return 0;
1502 }
1503
1504 static int spi_imx_poll_transfer(struct spi_device *spi,
1505                                  struct spi_transfer *transfer)
1506 {
1507         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1508         unsigned long timeout;
1509
1510         spi_imx->tx_buf = transfer->tx_buf;
1511         spi_imx->rx_buf = transfer->rx_buf;
1512         spi_imx->count = transfer->len;
1513         spi_imx->txfifo = 0;
1514         spi_imx->remainder = 0;
1515
1516         /* fill in the fifo before timeout calculations if we are
1517          * interrupted here, then the data is getting transferred by
1518          * the HW while we are interrupted
1519          */
1520         spi_imx_push(spi_imx);
1521
1522         timeout = spi_imx_calculate_timeout(spi_imx, transfer->len) + jiffies;
1523         while (spi_imx->txfifo) {
1524                 /* RX */
1525                 while (spi_imx->txfifo &&
1526                        spi_imx->devtype_data->rx_available(spi_imx)) {
1527                         spi_imx->rx(spi_imx);
1528                         spi_imx->txfifo--;
1529                 }
1530
1531                 /* TX */
1532                 if (spi_imx->count) {
1533                         spi_imx_push(spi_imx);
1534                         continue;
1535                 }
1536
1537                 if (spi_imx->txfifo &&
1538                     time_after(jiffies, timeout)) {
1539
1540                         dev_err_ratelimited(&spi->dev,
1541                                             "timeout period reached: jiffies: %lu- falling back to interrupt mode\n",
1542                                             jiffies - timeout);
1543
1544                         /* fall back to interrupt mode */
1545                         return spi_imx_pio_transfer(spi, transfer);
1546                 }
1547         }
1548
1549         return 0;
1550 }
1551
1552 static int spi_imx_pio_transfer_slave(struct spi_device *spi,
1553                                       struct spi_transfer *transfer)
1554 {
1555         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1556         int ret = 0;
1557
1558         if (is_imx53_ecspi(spi_imx) &&
1559             transfer->len > MX53_MAX_TRANSFER_BYTES) {
1560                 dev_err(&spi->dev, "Transaction too big, max size is %d bytes\n",
1561                         MX53_MAX_TRANSFER_BYTES);
1562                 return -EMSGSIZE;
1563         }
1564
1565         spi_imx->tx_buf = transfer->tx_buf;
1566         spi_imx->rx_buf = transfer->rx_buf;
1567         spi_imx->count = transfer->len;
1568         spi_imx->txfifo = 0;
1569         spi_imx->remainder = 0;
1570
1571         reinit_completion(&spi_imx->xfer_done);
1572         spi_imx->slave_aborted = false;
1573
1574         spi_imx_push(spi_imx);
1575
1576         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);
1577
1578         if (wait_for_completion_interruptible(&spi_imx->xfer_done) ||
1579             spi_imx->slave_aborted) {
1580                 dev_dbg(&spi->dev, "interrupted\n");
1581                 ret = -EINTR;
1582         }
1583
1584         /* ecspi has a HW issue when works in Slave mode,
1585          * after 64 words writtern to TXFIFO, even TXFIFO becomes empty,
1586          * ECSPI_TXDATA keeps shift out the last word data,
1587          * so we have to disable ECSPI when in slave mode after the
1588          * transfer completes
1589          */
1590         if (spi_imx->devtype_data->disable)
1591                 spi_imx->devtype_data->disable(spi_imx);
1592
1593         return ret;
1594 }
1595
1596 static int spi_imx_transfer_one(struct spi_controller *controller,
1597                                 struct spi_device *spi,
1598                                 struct spi_transfer *transfer)
1599 {
1600         struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1601         unsigned long hz_per_byte, byte_limit;
1602
1603         spi_imx_setupxfer(spi, transfer);
1604         transfer->effective_speed_hz = spi_imx->spi_bus_clk;
1605
1606         /* flush rxfifo before transfer */
1607         while (spi_imx->devtype_data->rx_available(spi_imx))
1608                 readl(spi_imx->base + MXC_CSPIRXDATA);
1609
1610         if (spi_imx->slave_mode)
1611                 return spi_imx_pio_transfer_slave(spi, transfer);
1612
1613         /*
1614          * If we decided in spi_imx_can_dma() that we want to do a DMA
1615          * transfer, the SPI transfer has already been mapped, so we
1616          * have to do the DMA transfer here.
1617          */
1618         if (spi_imx->usedma)
1619                 return spi_imx_dma_transfer(spi_imx, transfer);
1620         /*
1621          * Calculate the estimated time in us the transfer runs. Find
1622          * the number of Hz per byte per polling limit.
1623          */
1624         hz_per_byte = polling_limit_us ? ((8 + 4) * USEC_PER_SEC) / polling_limit_us : 0;
1625         byte_limit = hz_per_byte ? transfer->effective_speed_hz / hz_per_byte : 1;
1626
1627         /* run in polling mode for short transfers */
1628         if (transfer->len < byte_limit)
1629                 return spi_imx_poll_transfer(spi, transfer);
1630
1631         return spi_imx_pio_transfer(spi, transfer);
1632 }
1633
1634 static int spi_imx_setup(struct spi_device *spi)
1635 {
1636         dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
1637                  spi->mode, spi->bits_per_word, spi->max_speed_hz);
1638
1639         return 0;
1640 }
1641
1642 static void spi_imx_cleanup(struct spi_device *spi)
1643 {
1644 }
1645
1646 static int
1647 spi_imx_prepare_message(struct spi_controller *controller, struct spi_message *msg)
1648 {
1649         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1650         int ret;
1651
1652         ret = pm_runtime_resume_and_get(spi_imx->dev);
1653         if (ret < 0) {
1654                 dev_err(spi_imx->dev, "failed to enable clock\n");
1655                 return ret;
1656         }
1657
1658         ret = spi_imx->devtype_data->prepare_message(spi_imx, msg);
1659         if (ret) {
1660                 pm_runtime_mark_last_busy(spi_imx->dev);
1661                 pm_runtime_put_autosuspend(spi_imx->dev);
1662         }
1663
1664         return ret;
1665 }
1666
1667 static int
1668 spi_imx_unprepare_message(struct spi_controller *controller, struct spi_message *msg)
1669 {
1670         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1671
1672         pm_runtime_mark_last_busy(spi_imx->dev);
1673         pm_runtime_put_autosuspend(spi_imx->dev);
1674         return 0;
1675 }
1676
1677 static int spi_imx_slave_abort(struct spi_controller *controller)
1678 {
1679         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1680
1681         spi_imx->slave_aborted = true;
1682         complete(&spi_imx->xfer_done);
1683
1684         return 0;
1685 }
1686
1687 static int spi_imx_probe(struct platform_device *pdev)
1688 {
1689         struct device_node *np = pdev->dev.of_node;
1690         struct spi_controller *controller;
1691         struct spi_imx_data *spi_imx;
1692         struct resource *res;
1693         int ret, irq, spi_drctl;
1694         const struct spi_imx_devtype_data *devtype_data =
1695                         of_device_get_match_data(&pdev->dev);
1696         bool slave_mode;
1697         u32 val;
1698
1699         slave_mode = devtype_data->has_slavemode &&
1700                         of_property_read_bool(np, "spi-slave");
1701         if (slave_mode)
1702                 controller = spi_alloc_slave(&pdev->dev,
1703                                              sizeof(struct spi_imx_data));
1704         else
1705                 controller = spi_alloc_master(&pdev->dev,
1706                                               sizeof(struct spi_imx_data));
1707         if (!controller)
1708                 return -ENOMEM;
1709
1710         ret = of_property_read_u32(np, "fsl,spi-rdy-drctl", &spi_drctl);
1711         if ((ret < 0) || (spi_drctl >= 0x3)) {
1712                 /* '11' is reserved */
1713                 spi_drctl = 0;
1714         }
1715
1716         platform_set_drvdata(pdev, controller);
1717
1718         controller->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
1719         controller->bus_num = np ? -1 : pdev->id;
1720         controller->use_gpio_descriptors = true;
1721
1722         spi_imx = spi_controller_get_devdata(controller);
1723         spi_imx->controller = controller;
1724         spi_imx->dev = &pdev->dev;
1725         spi_imx->slave_mode = slave_mode;
1726
1727         spi_imx->devtype_data = devtype_data;
1728
1729         /*
1730          * Get number of chip selects from device properties. This can be
1731          * coming from device tree or boardfiles, if it is not defined,
1732          * a default value of 3 chip selects will be used, as all the legacy
1733          * board files have <= 3 chip selects.
1734          */
1735         if (!device_property_read_u32(&pdev->dev, "num-cs", &val))
1736                 controller->num_chipselect = val;
1737         else
1738                 controller->num_chipselect = 3;
1739
1740         spi_imx->controller->transfer_one = spi_imx_transfer_one;
1741         spi_imx->controller->setup = spi_imx_setup;
1742         spi_imx->controller->cleanup = spi_imx_cleanup;
1743         spi_imx->controller->prepare_message = spi_imx_prepare_message;
1744         spi_imx->controller->unprepare_message = spi_imx_unprepare_message;
1745         spi_imx->controller->slave_abort = spi_imx_slave_abort;
1746         spi_imx->controller->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_NO_CS;
1747
1748         if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||
1749             is_imx53_ecspi(spi_imx))
1750                 spi_imx->controller->mode_bits |= SPI_LOOP | SPI_READY;
1751
1752         if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx))
1753                 spi_imx->controller->mode_bits |= SPI_RX_CPHA_FLIP;
1754
1755         if (is_imx51_ecspi(spi_imx) &&
1756             device_property_read_u32(&pdev->dev, "cs-gpios", NULL))
1757                 /*
1758                  * When using HW-CS implementing SPI_CS_WORD can be done by just
1759                  * setting the burst length to the word size. This is
1760                  * considerably faster than manually controlling the CS.
1761                  */
1762                 spi_imx->controller->mode_bits |= SPI_CS_WORD;
1763
1764         spi_imx->spi_drctl = spi_drctl;
1765
1766         init_completion(&spi_imx->xfer_done);
1767
1768         spi_imx->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1769         if (IS_ERR(spi_imx->base)) {
1770                 ret = PTR_ERR(spi_imx->base);
1771                 goto out_controller_put;
1772         }
1773         spi_imx->base_phys = res->start;
1774
1775         irq = platform_get_irq(pdev, 0);
1776         if (irq < 0) {
1777                 ret = irq;
1778                 goto out_controller_put;
1779         }
1780
1781         ret = devm_request_irq(&pdev->dev, irq, spi_imx_isr, 0,
1782                                dev_name(&pdev->dev), spi_imx);
1783         if (ret) {
1784                 dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
1785                 goto out_controller_put;
1786         }
1787
1788         spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1789         if (IS_ERR(spi_imx->clk_ipg)) {
1790                 ret = PTR_ERR(spi_imx->clk_ipg);
1791                 goto out_controller_put;
1792         }
1793
1794         spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");
1795         if (IS_ERR(spi_imx->clk_per)) {
1796                 ret = PTR_ERR(spi_imx->clk_per);
1797                 goto out_controller_put;
1798         }
1799
1800         ret = clk_prepare_enable(spi_imx->clk_per);
1801         if (ret)
1802                 goto out_controller_put;
1803
1804         ret = clk_prepare_enable(spi_imx->clk_ipg);
1805         if (ret)
1806                 goto out_put_per;
1807
1808         pm_runtime_set_autosuspend_delay(spi_imx->dev, MXC_RPM_TIMEOUT);
1809         pm_runtime_use_autosuspend(spi_imx->dev);
1810         pm_runtime_get_noresume(spi_imx->dev);
1811         pm_runtime_set_active(spi_imx->dev);
1812         pm_runtime_enable(spi_imx->dev);
1813
1814         spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);
1815         /*
1816          * Only validated on i.mx35 and i.mx6 now, can remove the constraint
1817          * if validated on other chips.
1818          */
1819         if (spi_imx->devtype_data->has_dmamode) {
1820                 ret = spi_imx_sdma_init(&pdev->dev, spi_imx, controller);
1821                 if (ret == -EPROBE_DEFER)
1822                         goto out_runtime_pm_put;
1823
1824                 if (ret < 0)
1825                         dev_dbg(&pdev->dev, "dma setup error %d, use pio\n",
1826                                 ret);
1827         }
1828
1829         spi_imx->devtype_data->reset(spi_imx);
1830
1831         spi_imx->devtype_data->intctrl(spi_imx, 0);
1832
1833         controller->dev.of_node = pdev->dev.of_node;
1834         ret = spi_register_controller(controller);
1835         if (ret) {
1836                 dev_err_probe(&pdev->dev, ret, "register controller failed\n");
1837                 goto out_register_controller;
1838         }
1839
1840         pm_runtime_mark_last_busy(spi_imx->dev);
1841         pm_runtime_put_autosuspend(spi_imx->dev);
1842
1843         return ret;
1844
1845 out_register_controller:
1846         if (spi_imx->devtype_data->has_dmamode)
1847                 spi_imx_sdma_exit(spi_imx);
1848 out_runtime_pm_put:
1849         pm_runtime_dont_use_autosuspend(spi_imx->dev);
1850         pm_runtime_set_suspended(&pdev->dev);
1851         pm_runtime_disable(spi_imx->dev);
1852
1853         clk_disable_unprepare(spi_imx->clk_ipg);
1854 out_put_per:
1855         clk_disable_unprepare(spi_imx->clk_per);
1856 out_controller_put:
1857         spi_controller_put(controller);
1858
1859         return ret;
1860 }
1861
1862 static void spi_imx_remove(struct platform_device *pdev)
1863 {
1864         struct spi_controller *controller = platform_get_drvdata(pdev);
1865         struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1866         int ret;
1867
1868         spi_unregister_controller(controller);
1869
1870         ret = pm_runtime_get_sync(spi_imx->dev);
1871         if (ret >= 0)
1872                 writel(0, spi_imx->base + MXC_CSPICTRL);
1873         else
1874                 dev_warn(spi_imx->dev, "failed to enable clock, skip hw disable\n");
1875
1876         pm_runtime_dont_use_autosuspend(spi_imx->dev);
1877         pm_runtime_put_sync(spi_imx->dev);
1878         pm_runtime_disable(spi_imx->dev);
1879
1880         spi_imx_sdma_exit(spi_imx);
1881 }
1882
1883 static int __maybe_unused spi_imx_runtime_resume(struct device *dev)
1884 {
1885         struct spi_controller *controller = dev_get_drvdata(dev);
1886         struct spi_imx_data *spi_imx;
1887         int ret;
1888
1889         spi_imx = spi_controller_get_devdata(controller);
1890
1891         ret = clk_prepare_enable(spi_imx->clk_per);
1892         if (ret)
1893                 return ret;
1894
1895         ret = clk_prepare_enable(spi_imx->clk_ipg);
1896         if (ret) {
1897                 clk_disable_unprepare(spi_imx->clk_per);
1898                 return ret;
1899         }
1900
1901         return 0;
1902 }
1903
1904 static int __maybe_unused spi_imx_runtime_suspend(struct device *dev)
1905 {
1906         struct spi_controller *controller = dev_get_drvdata(dev);
1907         struct spi_imx_data *spi_imx;
1908
1909         spi_imx = spi_controller_get_devdata(controller);
1910
1911         clk_disable_unprepare(spi_imx->clk_per);
1912         clk_disable_unprepare(spi_imx->clk_ipg);
1913
1914         return 0;
1915 }
1916
1917 static int __maybe_unused spi_imx_suspend(struct device *dev)
1918 {
1919         pinctrl_pm_select_sleep_state(dev);
1920         return 0;
1921 }
1922
1923 static int __maybe_unused spi_imx_resume(struct device *dev)
1924 {
1925         pinctrl_pm_select_default_state(dev);
1926         return 0;
1927 }
1928
1929 static const struct dev_pm_ops imx_spi_pm = {
1930         SET_RUNTIME_PM_OPS(spi_imx_runtime_suspend,
1931                                 spi_imx_runtime_resume, NULL)
1932         SET_SYSTEM_SLEEP_PM_OPS(spi_imx_suspend, spi_imx_resume)
1933 };
1934
1935 static struct platform_driver spi_imx_driver = {
1936         .driver = {
1937                    .name = DRIVER_NAME,
1938                    .of_match_table = spi_imx_dt_ids,
1939                    .pm = &imx_spi_pm,
1940         },
1941         .probe = spi_imx_probe,
1942         .remove_new = spi_imx_remove,
1943 };
1944 module_platform_driver(spi_imx_driver);
1945
1946 MODULE_DESCRIPTION("i.MX SPI Controller driver");
1947 MODULE_AUTHOR("Sascha Hauer, Pengutronix");
1948 MODULE_LICENSE("GPL");
1949 MODULE_ALIAS("platform:" DRIVER_NAME);