drivers/spi/spi-mt65xx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2015 MediaTek Inc.
   4  * Author: Leilk Liu <[email protected]>
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/device.h>
   9 #include <linux/err.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/io.h>
  12 #include <linux/ioport.h>
  13 #include <linux/module.h>
  14 #include <linux/of.h>
  15 #include <linux/gpio/consumer.h>
  16 #include <linux/pinctrl/consumer.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/platform_data/spi-mt65xx.h>
  19 #include <linux/pm_runtime.h>
  20 #include <linux/spi/spi.h>
  21 #include <linux/spi/spi-mem.h>
  22 #include <linux/dma-mapping.h>
  23
  24 #define SPI_CFG0_REG                    0x0000
  25 #define SPI_CFG1_REG                    0x0004
  26 #define SPI_TX_SRC_REG                  0x0008
  27 #define SPI_RX_DST_REG                  0x000c
  28 #define SPI_TX_DATA_REG                 0x0010
  29 #define SPI_RX_DATA_REG                 0x0014
  30 #define SPI_CMD_REG                     0x0018
  31 #define SPI_STATUS0_REG                 0x001c
  32 #define SPI_PAD_SEL_REG                 0x0024
  33 #define SPI_CFG2_REG                    0x0028
  34 #define SPI_TX_SRC_REG_64               0x002c
  35 #define SPI_RX_DST_REG_64               0x0030
  36 #define SPI_CFG3_IPM_REG                0x0040
  37
  38 #define SPI_CFG0_SCK_HIGH_OFFSET        0
  39 #define SPI_CFG0_SCK_LOW_OFFSET         8
  40 #define SPI_CFG0_CS_HOLD_OFFSET         16
  41 #define SPI_CFG0_CS_SETUP_OFFSET        24
  42 #define SPI_ADJUST_CFG0_CS_HOLD_OFFSET  0
  43 #define SPI_ADJUST_CFG0_CS_SETUP_OFFSET 16
  44
  45 #define SPI_CFG1_CS_IDLE_OFFSET         0
  46 #define SPI_CFG1_PACKET_LOOP_OFFSET     8
  47 #define SPI_CFG1_PACKET_LENGTH_OFFSET   16
  48 #define SPI_CFG1_GET_TICK_DLY_OFFSET    29
  49 #define SPI_CFG1_GET_TICK_DLY_OFFSET_V1 30
  50
  51 #define SPI_CFG1_GET_TICK_DLY_MASK      0xe0000000
  52 #define SPI_CFG1_GET_TICK_DLY_MASK_V1   0xc0000000
  53
  54 #define SPI_CFG1_CS_IDLE_MASK           0xff
  55 #define SPI_CFG1_PACKET_LOOP_MASK       0xff00
  56 #define SPI_CFG1_PACKET_LENGTH_MASK     0x3ff0000
  57 #define SPI_CFG1_IPM_PACKET_LENGTH_MASK GENMASK(31, 16)
  58 #define SPI_CFG2_SCK_HIGH_OFFSET        0
  59 #define SPI_CFG2_SCK_LOW_OFFSET         16
  60
  61 #define SPI_CMD_ACT                     BIT(0)
  62 #define SPI_CMD_RESUME                  BIT(1)
  63 #define SPI_CMD_RST                     BIT(2)
  64 #define SPI_CMD_PAUSE_EN                BIT(4)
  65 #define SPI_CMD_DEASSERT                BIT(5)
  66 #define SPI_CMD_SAMPLE_SEL              BIT(6)
  67 #define SPI_CMD_CS_POL                  BIT(7)
  68 #define SPI_CMD_CPHA                    BIT(8)
  69 #define SPI_CMD_CPOL                    BIT(9)
  70 #define SPI_CMD_RX_DMA                  BIT(10)
  71 #define SPI_CMD_TX_DMA                  BIT(11)
  72 #define SPI_CMD_TXMSBF                  BIT(12)
  73 #define SPI_CMD_RXMSBF                  BIT(13)
  74 #define SPI_CMD_RX_ENDIAN               BIT(14)
  75 #define SPI_CMD_TX_ENDIAN               BIT(15)
  76 #define SPI_CMD_FINISH_IE               BIT(16)
  77 #define SPI_CMD_PAUSE_IE                BIT(17)
  78 #define SPI_CMD_IPM_NONIDLE_MODE        BIT(19)
  79 #define SPI_CMD_IPM_SPIM_LOOP           BIT(21)
  80 #define SPI_CMD_IPM_GET_TICKDLY_OFFSET  22
  81
  82 #define SPI_CMD_IPM_GET_TICKDLY_MASK    GENMASK(24, 22)
  83
  84 #define PIN_MODE_CFG(x) ((x) / 2)
  85
  86 #define SPI_CFG3_IPM_HALF_DUPLEX_DIR    BIT(2)
  87 #define SPI_CFG3_IPM_HALF_DUPLEX_EN     BIT(3)
  88 #define SPI_CFG3_IPM_XMODE_EN           BIT(4)
  89 #define SPI_CFG3_IPM_NODATA_FLAG        BIT(5)
  90 #define SPI_CFG3_IPM_CMD_BYTELEN_OFFSET 8
  91 #define SPI_CFG3_IPM_ADDR_BYTELEN_OFFSET 12
  92
  93 #define SPI_CFG3_IPM_CMD_PIN_MODE_MASK  GENMASK(1, 0)
  94 #define SPI_CFG3_IPM_CMD_BYTELEN_MASK   GENMASK(11, 8)
  95 #define SPI_CFG3_IPM_ADDR_BYTELEN_MASK  GENMASK(15, 12)
  96
  97 #define MT8173_SPI_MAX_PAD_SEL          3
  98
  99 #define MTK_SPI_PAUSE_INT_STATUS        0x2
 100
 101 #define MTK_SPI_MAX_FIFO_SIZE           32U
 102 #define MTK_SPI_PACKET_SIZE             1024
 103 #define MTK_SPI_IPM_PACKET_SIZE         SZ_64K
 104 #define MTK_SPI_IPM_PACKET_LOOP         SZ_256
 105
 106 #define MTK_SPI_IDLE                    0
 107 #define MTK_SPI_PAUSED                  1
 108
 109 #define MTK_SPI_32BITS_MASK             (0xffffffff)
 110
 111 #define DMA_ADDR_EXT_BITS               (36)
 112 #define DMA_ADDR_DEF_BITS               (32)
 113
 114 /**
 115  * struct mtk_spi_compatible - device data structure
 116  * @need_pad_sel:       Enable pad (pins) selection in SPI controller
 117  * @must_tx:            Must explicitly send dummy TX bytes to do RX only transfer
 118  * @enhance_timing:     Enable adjusting cfg register to enhance time accuracy
 119  * @dma_ext:            DMA address extension supported
 120  * @no_need_unprepare:  Don't unprepare the SPI clk during runtime
 121  * @ipm_design:         Adjust/extend registers to support IPM design IP features
 122  */
 123 struct mtk_spi_compatible {
 124         bool need_pad_sel;
 125         bool must_tx;
 126         bool enhance_timing;
 127         bool dma_ext;
 128         bool no_need_unprepare;
 129         bool ipm_design;
 130 };
 131
 132 /**
 133  * struct mtk_spi - SPI driver instance
 134  * @base:               Start address of the SPI controller registers
 135  * @state:              SPI controller state
 136  * @pad_num:            Number of pad_sel entries
 137  * @pad_sel:            Groups of pins to select
 138  * @parent_clk:         Parent of sel_clk
 139  * @sel_clk:            SPI host mux clock
 140  * @spi_clk:            Peripheral clock
 141  * @spi_hclk:           AHB bus clock
 142  * @cur_transfer:       Currently processed SPI transfer
 143  * @xfer_len:           Number of bytes to transfer
 144  * @num_xfered:         Number of transferred bytes
 145  * @tx_sgl:             TX transfer scatterlist
 146  * @rx_sgl:             RX transfer scatterlist
 147  * @tx_sgl_len:         Size of TX DMA transfer
 148  * @rx_sgl_len:         Size of RX DMA transfer
 149  * @dev_comp:           Device data structure
 150  * @spi_clk_hz:         Current SPI clock in Hz
 151  * @spimem_done:        SPI-MEM operation completion
 152  * @use_spimem:         Enables SPI-MEM
 153  * @dev:                Device pointer
 154  * @tx_dma:             DMA start for SPI-MEM TX
 155  * @rx_dma:             DMA start for SPI-MEM RX
 156  */
 157 struct mtk_spi {
 158         void __iomem *base;
 159         u32 state;
 160         int pad_num;
 161         u32 *pad_sel;
 162         struct clk *parent_clk, *sel_clk, *spi_clk, *spi_hclk;
 163         struct spi_transfer *cur_transfer;
 164         u32 xfer_len;
 165         u32 num_xfered;
 166         struct scatterlist *tx_sgl, *rx_sgl;
 167         u32 tx_sgl_len, rx_sgl_len;
 168         const struct mtk_spi_compatible *dev_comp;
 169         u32 spi_clk_hz;
 170         struct completion spimem_done;
 171         bool use_spimem;
 172         struct device *dev;
 173         dma_addr_t tx_dma;
 174         dma_addr_t rx_dma;
 175 };
 176
 177 static const struct mtk_spi_compatible mtk_common_compat;
 178
 179 static const struct mtk_spi_compatible mt2712_compat = {
 180         .must_tx = true,
 181 };
 182
 183 static const struct mtk_spi_compatible mtk_ipm_compat = {
 184         .enhance_timing = true,
 185         .dma_ext = true,
 186         .ipm_design = true,
 187 };
 188
 189 static const struct mtk_spi_compatible mt6765_compat = {
 190         .need_pad_sel = true,
 191         .must_tx = true,
 192         .enhance_timing = true,
 193         .dma_ext = true,
 194 };
 195
 196 static const struct mtk_spi_compatible mt7622_compat = {
 197         .must_tx = true,
 198         .enhance_timing = true,
 199 };
 200
 201 static const struct mtk_spi_compatible mt8173_compat = {
 202         .need_pad_sel = true,
 203         .must_tx = true,
 204 };
 205
 206 static const struct mtk_spi_compatible mt8183_compat = {
 207         .need_pad_sel = true,
 208         .must_tx = true,
 209         .enhance_timing = true,
 210 };
 211
 212 static const struct mtk_spi_compatible mt6893_compat = {
 213         .need_pad_sel = true,
 214         .must_tx = true,
 215         .enhance_timing = true,
 216         .dma_ext = true,
 217         .no_need_unprepare = true,
 218 };
 219
 220 /*
 221  * A piece of default chip info unless the platform
 222  * supplies it.
 223  */
 224 static const struct mtk_chip_config mtk_default_chip_info = {
 225         .sample_sel = 0,
 226         .tick_delay = 0,
 227 };
 228
 229 static const struct of_device_id mtk_spi_of_match[] = {
 230         { .compatible = "mediatek,spi-ipm",
 231                 .data = (void *)&mtk_ipm_compat,
 232         },
 233         { .compatible = "mediatek,mt2701-spi",
 234                 .data = (void *)&mtk_common_compat,
 235         },
 236         { .compatible = "mediatek,mt2712-spi",
 237                 .data = (void *)&mt2712_compat,
 238         },
 239         { .compatible = "mediatek,mt6589-spi",
 240                 .data = (void *)&mtk_common_compat,
 241         },
 242         { .compatible = "mediatek,mt6765-spi",
 243                 .data = (void *)&mt6765_compat,
 244         },
 245         { .compatible = "mediatek,mt7622-spi",
 246                 .data = (void *)&mt7622_compat,
 247         },
 248         { .compatible = "mediatek,mt7629-spi",
 249                 .data = (void *)&mt7622_compat,
 250         },
 251         { .compatible = "mediatek,mt8135-spi",
 252                 .data = (void *)&mtk_common_compat,
 253         },
 254         { .compatible = "mediatek,mt8173-spi",
 255                 .data = (void *)&mt8173_compat,
 256         },
 257         { .compatible = "mediatek,mt8183-spi",
 258                 .data = (void *)&mt8183_compat,
 259         },
 260         { .compatible = "mediatek,mt8192-spi",
 261                 .data = (void *)&mt6765_compat,
 262         },
 263         { .compatible = "mediatek,mt6893-spi",
 264                 .data = (void *)&mt6893_compat,
 265         },
 266         {}
 267 };
 268 MODULE_DEVICE_TABLE(of, mtk_spi_of_match);
 269
 270 static void mtk_spi_reset(struct mtk_spi *mdata)
 271 {
 272         u32 reg_val;
 273
 274         /* set the software reset bit in SPI_CMD_REG. */
 275         reg_val = readl(mdata->base + SPI_CMD_REG);
 276         reg_val |= SPI_CMD_RST;
 277         writel(reg_val, mdata->base + SPI_CMD_REG);
 278
 279         reg_val = readl(mdata->base + SPI_CMD_REG);
 280         reg_val &= ~SPI_CMD_RST;
 281         writel(reg_val, mdata->base + SPI_CMD_REG);
 282 }
 283
 284 static int mtk_spi_set_hw_cs_timing(struct spi_device *spi)
 285 {
 286         struct mtk_spi *mdata = spi_controller_get_devdata(spi->controller);
 287         struct spi_delay *cs_setup = &spi->cs_setup;
 288         struct spi_delay *cs_hold = &spi->cs_hold;
 289         struct spi_delay *cs_inactive = &spi->cs_inactive;
 290         u32 setup, hold, inactive;
 291         u32 reg_val;
 292         int delay;
 293
 294         delay = spi_delay_to_ns(cs_setup, NULL);
 295         if (delay < 0)
 296                 return delay;
 297         setup = (delay * DIV_ROUND_UP(mdata->spi_clk_hz, 1000000)) / 1000;
 298
 299         delay = spi_delay_to_ns(cs_hold, NULL);
 300         if (delay < 0)
 301                 return delay;
 302         hold = (delay * DIV_ROUND_UP(mdata->spi_clk_hz, 1000000)) / 1000;
 303
 304         delay = spi_delay_to_ns(cs_inactive, NULL);
 305         if (delay < 0)
 306                 return delay;
 307         inactive = (delay * DIV_ROUND_UP(mdata->spi_clk_hz, 1000000)) / 1000;
 308
 309         if (hold || setup) {
 310                 reg_val = readl(mdata->base + SPI_CFG0_REG);
 311                 if (mdata->dev_comp->enhance_timing) {
 312                         if (hold) {
 313                                 hold = min_t(u32, hold, 0x10000);
 314                                 reg_val &= ~(0xffff << SPI_ADJUST_CFG0_CS_HOLD_OFFSET);
 315                                 reg_val |= (((hold - 1) & 0xffff)
 316                                         << SPI_ADJUST_CFG0_CS_HOLD_OFFSET);
 317                         }
 318                         if (setup) {
 319                                 setup = min_t(u32, setup, 0x10000);
 320                                 reg_val &= ~(0xffff << SPI_ADJUST_CFG0_CS_SETUP_OFFSET);
 321                                 reg_val |= (((setup - 1) & 0xffff)
 322                                         << SPI_ADJUST_CFG0_CS_SETUP_OFFSET);
 323                         }
 324                 } else {
 325                         if (hold) {
 326                                 hold = min_t(u32, hold, 0x100);
 327                                 reg_val &= ~(0xff << SPI_CFG0_CS_HOLD_OFFSET);
 328                                 reg_val |= (((hold - 1) & 0xff) << SPI_CFG0_CS_HOLD_OFFSET);
 329                         }
 330                         if (setup) {
 331                                 setup = min_t(u32, setup, 0x100);
 332                                 reg_val &= ~(0xff << SPI_CFG0_CS_SETUP_OFFSET);
 333                                 reg_val |= (((setup - 1) & 0xff)
 334                                         << SPI_CFG0_CS_SETUP_OFFSET);
 335                         }
 336                 }
 337                 writel(reg_val, mdata->base + SPI_CFG0_REG);
 338         }
 339
 340         if (inactive) {
 341                 inactive = min_t(u32, inactive, 0x100);
 342                 reg_val = readl(mdata->base + SPI_CFG1_REG);
 343                 reg_val &= ~SPI_CFG1_CS_IDLE_MASK;
 344                 reg_val |= (((inactive - 1) & 0xff) << SPI_CFG1_CS_IDLE_OFFSET);
 345                 writel(reg_val, mdata->base + SPI_CFG1_REG);
 346         }
 347
 348         return 0;
 349 }
 350
 351 static int mtk_spi_hw_init(struct spi_controller *host,
 352                            struct spi_device *spi)
 353 {
 354         u16 cpha, cpol;
 355         u32 reg_val;
 356         struct mtk_chip_config *chip_config = spi->controller_data;
 357         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 358
 359         cpha = spi->mode & SPI_CPHA ? 1 : 0;
 360         cpol = spi->mode & SPI_CPOL ? 1 : 0;
 361
 362         reg_val = readl(mdata->base + SPI_CMD_REG);
 363         if (mdata->dev_comp->ipm_design) {
 364                 /* SPI transfer without idle time until packet length done */
 365                 reg_val |= SPI_CMD_IPM_NONIDLE_MODE;
 366                 if (spi->mode & SPI_LOOP)
 367                         reg_val |= SPI_CMD_IPM_SPIM_LOOP;
 368                 else
 369                         reg_val &= ~SPI_CMD_IPM_SPIM_LOOP;
 370         }
 371
 372         if (cpha)
 373                 reg_val |= SPI_CMD_CPHA;
 374         else
 375                 reg_val &= ~SPI_CMD_CPHA;
 376         if (cpol)
 377                 reg_val |= SPI_CMD_CPOL;
 378         else
 379                 reg_val &= ~SPI_CMD_CPOL;
 380
 381         /* set the mlsbx and mlsbtx */
 382         if (spi->mode & SPI_LSB_FIRST) {
 383                 reg_val &= ~SPI_CMD_TXMSBF;
 384                 reg_val &= ~SPI_CMD_RXMSBF;
 385         } else {
 386                 reg_val |= SPI_CMD_TXMSBF;
 387                 reg_val |= SPI_CMD_RXMSBF;
 388         }
 389
 390         /* set the tx/rx endian */
 391 #ifdef __LITTLE_ENDIAN
 392         reg_val &= ~SPI_CMD_TX_ENDIAN;
 393         reg_val &= ~SPI_CMD_RX_ENDIAN;
 394 #else
 395         reg_val |= SPI_CMD_TX_ENDIAN;
 396         reg_val |= SPI_CMD_RX_ENDIAN;
 397 #endif
 398
 399         if (mdata->dev_comp->enhance_timing) {
 400                 /* set CS polarity */
 401                 if (spi->mode & SPI_CS_HIGH)
 402                         reg_val |= SPI_CMD_CS_POL;
 403                 else
 404                         reg_val &= ~SPI_CMD_CS_POL;
 405
 406                 if (chip_config->sample_sel)
 407                         reg_val |= SPI_CMD_SAMPLE_SEL;
 408                 else
 409                         reg_val &= ~SPI_CMD_SAMPLE_SEL;
 410         }
 411
 412         /* set finish and pause interrupt always enable */
 413         reg_val |= SPI_CMD_FINISH_IE | SPI_CMD_PAUSE_IE;
 414
 415         /* disable dma mode */
 416         reg_val &= ~(SPI_CMD_TX_DMA | SPI_CMD_RX_DMA);
 417
 418         /* disable deassert mode */
 419         reg_val &= ~SPI_CMD_DEASSERT;
 420
 421         writel(reg_val, mdata->base + SPI_CMD_REG);
 422
 423         /* pad select */
 424         if (mdata->dev_comp->need_pad_sel)
 425                 writel(mdata->pad_sel[spi_get_chipselect(spi, 0)],
 426                        mdata->base + SPI_PAD_SEL_REG);
 427
 428         /* tick delay */
 429         if (mdata->dev_comp->enhance_timing) {
 430                 if (mdata->dev_comp->ipm_design) {
 431                         reg_val = readl(mdata->base + SPI_CMD_REG);
 432                         reg_val &= ~SPI_CMD_IPM_GET_TICKDLY_MASK;
 433                         reg_val |= ((chip_config->tick_delay & 0x7)
 434                                     << SPI_CMD_IPM_GET_TICKDLY_OFFSET);
 435                         writel(reg_val, mdata->base + SPI_CMD_REG);
 436                 } else {
 437                         reg_val = readl(mdata->base + SPI_CFG1_REG);
 438                         reg_val &= ~SPI_CFG1_GET_TICK_DLY_MASK;
 439                         reg_val |= ((chip_config->tick_delay & 0x7)
 440                                     << SPI_CFG1_GET_TICK_DLY_OFFSET);
 441                         writel(reg_val, mdata->base + SPI_CFG1_REG);
 442                 }
 443         } else {
 444                 reg_val = readl(mdata->base + SPI_CFG1_REG);
 445                 reg_val &= ~SPI_CFG1_GET_TICK_DLY_MASK_V1;
 446                 reg_val |= ((chip_config->tick_delay & 0x3)
 447                             << SPI_CFG1_GET_TICK_DLY_OFFSET_V1);
 448                 writel(reg_val, mdata->base + SPI_CFG1_REG);
 449         }
 450
 451         /* set hw cs timing */
 452         mtk_spi_set_hw_cs_timing(spi);
 453         return 0;
 454 }
 455
 456 static int mtk_spi_prepare_message(struct spi_controller *host,
 457                                    struct spi_message *msg)
 458 {
 459         return mtk_spi_hw_init(host, msg->spi);
 460 }
 461
 462 static void mtk_spi_set_cs(struct spi_device *spi, bool enable)
 463 {
 464         u32 reg_val;
 465         struct mtk_spi *mdata = spi_controller_get_devdata(spi->controller);
 466
 467         if (spi->mode & SPI_CS_HIGH)
 468                 enable = !enable;
 469
 470         reg_val = readl(mdata->base + SPI_CMD_REG);
 471         if (!enable) {
 472                 reg_val |= SPI_CMD_PAUSE_EN;
 473                 writel(reg_val, mdata->base + SPI_CMD_REG);
 474         } else {
 475                 reg_val &= ~SPI_CMD_PAUSE_EN;
 476                 writel(reg_val, mdata->base + SPI_CMD_REG);
 477                 mdata->state = MTK_SPI_IDLE;
 478                 mtk_spi_reset(mdata);
 479         }
 480 }
 481
 482 static void mtk_spi_prepare_transfer(struct spi_controller *host,
 483                                      u32 speed_hz)
 484 {
 485         u32 div, sck_time, reg_val;
 486         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 487
 488         if (speed_hz < mdata->spi_clk_hz / 2)
 489                 div = DIV_ROUND_UP(mdata->spi_clk_hz, speed_hz);
 490         else
 491                 div = 1;
 492
 493         sck_time = (div + 1) / 2;
 494
 495         if (mdata->dev_comp->enhance_timing) {
 496                 reg_val = readl(mdata->base + SPI_CFG2_REG);
 497                 reg_val &= ~(0xffff << SPI_CFG2_SCK_HIGH_OFFSET);
 498                 reg_val |= (((sck_time - 1) & 0xffff)
 499                            << SPI_CFG2_SCK_HIGH_OFFSET);
 500                 reg_val &= ~(0xffff << SPI_CFG2_SCK_LOW_OFFSET);
 501                 reg_val |= (((sck_time - 1) & 0xffff)
 502                            << SPI_CFG2_SCK_LOW_OFFSET);
 503                 writel(reg_val, mdata->base + SPI_CFG2_REG);
 504         } else {
 505                 reg_val = readl(mdata->base + SPI_CFG0_REG);
 506                 reg_val &= ~(0xff << SPI_CFG0_SCK_HIGH_OFFSET);
 507                 reg_val |= (((sck_time - 1) & 0xff)
 508                            << SPI_CFG0_SCK_HIGH_OFFSET);
 509                 reg_val &= ~(0xff << SPI_CFG0_SCK_LOW_OFFSET);
 510                 reg_val |= (((sck_time - 1) & 0xff) << SPI_CFG0_SCK_LOW_OFFSET);
 511                 writel(reg_val, mdata->base + SPI_CFG0_REG);
 512         }
 513 }
 514
 515 static void mtk_spi_setup_packet(struct spi_controller *host)
 516 {
 517         u32 packet_size, packet_loop, reg_val;
 518         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 519
 520         if (mdata->dev_comp->ipm_design)
 521                 packet_size = min_t(u32,
 522                                     mdata->xfer_len,
 523                                     MTK_SPI_IPM_PACKET_SIZE);
 524         else
 525                 packet_size = min_t(u32,
 526                                     mdata->xfer_len,
 527                                     MTK_SPI_PACKET_SIZE);
 528
 529         packet_loop = mdata->xfer_len / packet_size;
 530
 531         reg_val = readl(mdata->base + SPI_CFG1_REG);
 532         if (mdata->dev_comp->ipm_design)
 533                 reg_val &= ~SPI_CFG1_IPM_PACKET_LENGTH_MASK;
 534         else
 535                 reg_val &= ~SPI_CFG1_PACKET_LENGTH_MASK;
 536         reg_val |= (packet_size - 1) << SPI_CFG1_PACKET_LENGTH_OFFSET;
 537         reg_val &= ~SPI_CFG1_PACKET_LOOP_MASK;
 538         reg_val |= (packet_loop - 1) << SPI_CFG1_PACKET_LOOP_OFFSET;
 539         writel(reg_val, mdata->base + SPI_CFG1_REG);
 540 }
 541
 542 static void mtk_spi_enable_transfer(struct spi_controller *host)
 543 {
 544         u32 cmd;
 545         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 546
 547         cmd = readl(mdata->base + SPI_CMD_REG);
 548         if (mdata->state == MTK_SPI_IDLE)
 549                 cmd |= SPI_CMD_ACT;
 550         else
 551                 cmd |= SPI_CMD_RESUME;
 552         writel(cmd, mdata->base + SPI_CMD_REG);
 553 }
 554
 555 static int mtk_spi_get_mult_delta(struct mtk_spi *mdata, u32 xfer_len)
 556 {
 557         u32 mult_delta = 0;
 558
 559         if (mdata->dev_comp->ipm_design) {
 560                 if (xfer_len > MTK_SPI_IPM_PACKET_SIZE)
 561                         mult_delta = xfer_len % MTK_SPI_IPM_PACKET_SIZE;
 562         } else {
 563                 if (xfer_len > MTK_SPI_PACKET_SIZE)
 564                         mult_delta = xfer_len % MTK_SPI_PACKET_SIZE;
 565         }
 566
 567         return mult_delta;
 568 }
 569
 570 static void mtk_spi_update_mdata_len(struct spi_controller *host)
 571 {
 572         int mult_delta;
 573         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 574
 575         if (mdata->tx_sgl_len && mdata->rx_sgl_len) {
 576                 if (mdata->tx_sgl_len > mdata->rx_sgl_len) {
 577                         mult_delta = mtk_spi_get_mult_delta(mdata, mdata->rx_sgl_len);
 578                         mdata->xfer_len = mdata->rx_sgl_len - mult_delta;
 579                         mdata->rx_sgl_len = mult_delta;
 580                         mdata->tx_sgl_len -= mdata->xfer_len;
 581                 } else {
 582                         mult_delta = mtk_spi_get_mult_delta(mdata, mdata->tx_sgl_len);
 583                         mdata->xfer_len = mdata->tx_sgl_len - mult_delta;
 584                         mdata->tx_sgl_len = mult_delta;
 585                         mdata->rx_sgl_len -= mdata->xfer_len;
 586                 }
 587         } else if (mdata->tx_sgl_len) {
 588                 mult_delta = mtk_spi_get_mult_delta(mdata, mdata->tx_sgl_len);
 589                 mdata->xfer_len = mdata->tx_sgl_len - mult_delta;
 590                 mdata->tx_sgl_len = mult_delta;
 591         } else if (mdata->rx_sgl_len) {
 592                 mult_delta = mtk_spi_get_mult_delta(mdata, mdata->rx_sgl_len);
 593                 mdata->xfer_len = mdata->rx_sgl_len - mult_delta;
 594                 mdata->rx_sgl_len = mult_delta;
 595         }
 596 }
 597
 598 static void mtk_spi_setup_dma_addr(struct spi_controller *host,
 599                                    struct spi_transfer *xfer)
 600 {
 601         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 602
 603         if (mdata->tx_sgl) {
 604                 writel((u32)(xfer->tx_dma & MTK_SPI_32BITS_MASK),
 605                        mdata->base + SPI_TX_SRC_REG);
 606 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 607                 if (mdata->dev_comp->dma_ext)
 608                         writel((u32)(xfer->tx_dma >> 32),
 609                                mdata->base + SPI_TX_SRC_REG_64);
 610 #endif
 611         }
 612
 613         if (mdata->rx_sgl) {
 614                 writel((u32)(xfer->rx_dma & MTK_SPI_32BITS_MASK),
 615                        mdata->base + SPI_RX_DST_REG);
 616 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 617                 if (mdata->dev_comp->dma_ext)
 618                         writel((u32)(xfer->rx_dma >> 32),
 619                                mdata->base + SPI_RX_DST_REG_64);
 620 #endif
 621         }
 622 }
 623
 624 static int mtk_spi_fifo_transfer(struct spi_controller *host,
 625                                  struct spi_device *spi,
 626                                  struct spi_transfer *xfer)
 627 {
 628         int cnt, remainder;
 629         u32 reg_val;
 630         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 631
 632         mdata->cur_transfer = xfer;
 633         mdata->xfer_len = min(MTK_SPI_MAX_FIFO_SIZE, xfer->len);
 634         mdata->num_xfered = 0;
 635         mtk_spi_prepare_transfer(host, xfer->speed_hz);
 636         mtk_spi_setup_packet(host);
 637
 638         if (xfer->tx_buf) {
 639                 cnt = xfer->len / 4;
 640                 iowrite32_rep(mdata->base + SPI_TX_DATA_REG, xfer->tx_buf, cnt);
 641                 remainder = xfer->len % 4;
 642                 if (remainder > 0) {
 643                         reg_val = 0;
 644                         memcpy(&reg_val, xfer->tx_buf + (cnt * 4), remainder);
 645                         writel(reg_val, mdata->base + SPI_TX_DATA_REG);
 646                 }
 647         }
 648
 649         mtk_spi_enable_transfer(host);
 650
 651         return 1;
 652 }
 653
 654 static int mtk_spi_dma_transfer(struct spi_controller *host,
 655                                 struct spi_device *spi,
 656                                 struct spi_transfer *xfer)
 657 {
 658         int cmd;
 659         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 660
 661         mdata->tx_sgl = NULL;
 662         mdata->rx_sgl = NULL;
 663         mdata->tx_sgl_len = 0;
 664         mdata->rx_sgl_len = 0;
 665         mdata->cur_transfer = xfer;
 666         mdata->num_xfered = 0;
 667
 668         mtk_spi_prepare_transfer(host, xfer->speed_hz);
 669
 670         cmd = readl(mdata->base + SPI_CMD_REG);
 671         if (xfer->tx_buf)
 672                 cmd |= SPI_CMD_TX_DMA;
 673         if (xfer->rx_buf)
 674                 cmd |= SPI_CMD_RX_DMA;
 675         writel(cmd, mdata->base + SPI_CMD_REG);
 676
 677         if (xfer->tx_buf)
 678                 mdata->tx_sgl = xfer->tx_sg.sgl;
 679         if (xfer->rx_buf)
 680                 mdata->rx_sgl = xfer->rx_sg.sgl;
 681
 682         if (mdata->tx_sgl) {
 683                 xfer->tx_dma = sg_dma_address(mdata->tx_sgl);
 684                 mdata->tx_sgl_len = sg_dma_len(mdata->tx_sgl);
 685         }
 686         if (mdata->rx_sgl) {
 687                 xfer->rx_dma = sg_dma_address(mdata->rx_sgl);
 688                 mdata->rx_sgl_len = sg_dma_len(mdata->rx_sgl);
 689         }
 690
 691         mtk_spi_update_mdata_len(host);
 692         mtk_spi_setup_packet(host);
 693         mtk_spi_setup_dma_addr(host, xfer);
 694         mtk_spi_enable_transfer(host);
 695
 696         return 1;
 697 }
 698
 699 static int mtk_spi_transfer_one(struct spi_controller *host,
 700                                 struct spi_device *spi,
 701                                 struct spi_transfer *xfer)
 702 {
 703         struct mtk_spi *mdata = spi_controller_get_devdata(spi->controller);
 704         u32 reg_val = 0;
 705
 706         /* prepare xfer direction and duplex mode */
 707         if (mdata->dev_comp->ipm_design) {
 708                 if (!xfer->tx_buf || !xfer->rx_buf) {
 709                         reg_val |= SPI_CFG3_IPM_HALF_DUPLEX_EN;
 710                         if (xfer->rx_buf)
 711                                 reg_val |= SPI_CFG3_IPM_HALF_DUPLEX_DIR;
 712                 }
 713                 writel(reg_val, mdata->base + SPI_CFG3_IPM_REG);
 714         }
 715
 716         if (host->can_dma(host, spi, xfer))
 717                 return mtk_spi_dma_transfer(host, spi, xfer);
 718         else
 719                 return mtk_spi_fifo_transfer(host, spi, xfer);
 720 }
 721
 722 static bool mtk_spi_can_dma(struct spi_controller *host,
 723                             struct spi_device *spi,
 724                             struct spi_transfer *xfer)
 725 {
 726         /* Buffers for DMA transactions must be 4-byte aligned */
 727         return (xfer->len > MTK_SPI_MAX_FIFO_SIZE &&
 728                 (unsigned long)xfer->tx_buf % 4 == 0 &&
 729                 (unsigned long)xfer->rx_buf % 4 == 0);
 730 }
 731
 732 static int mtk_spi_setup(struct spi_device *spi)
 733 {
 734         struct mtk_spi *mdata = spi_controller_get_devdata(spi->controller);
 735
 736         if (!spi->controller_data)
 737                 spi->controller_data = (void *)&mtk_default_chip_info;
 738
 739         if (mdata->dev_comp->need_pad_sel && spi_get_csgpiod(spi, 0))
 740                 /* CS de-asserted, gpiolib will handle inversion */
 741                 gpiod_direction_output(spi_get_csgpiod(spi, 0), 0);
 742
 743         return 0;
 744 }
 745
 746 static irqreturn_t mtk_spi_interrupt_thread(int irq, void *dev_id)
 747 {
 748         u32 cmd, reg_val, cnt, remainder, len;
 749         struct spi_controller *host = dev_id;
 750         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 751         struct spi_transfer *xfer = mdata->cur_transfer;
 752
 753         if (!host->can_dma(host, NULL, xfer)) {
 754                 if (xfer->rx_buf) {
 755                         cnt = mdata->xfer_len / 4;
 756                         ioread32_rep(mdata->base + SPI_RX_DATA_REG,
 757                                      xfer->rx_buf + mdata->num_xfered, cnt);
 758                         remainder = mdata->xfer_len % 4;
 759                         if (remainder > 0) {
 760                                 reg_val = readl(mdata->base + SPI_RX_DATA_REG);
 761                                 memcpy(xfer->rx_buf + (cnt * 4) + mdata->num_xfered,
 762                                         &reg_val,
 763                                         remainder);
 764                         }
 765                 }
 766
 767                 mdata->num_xfered += mdata->xfer_len;
 768                 if (mdata->num_xfered == xfer->len) {
 769                         spi_finalize_current_transfer(host);
 770                         return IRQ_HANDLED;
 771                 }
 772
 773                 len = xfer->len - mdata->num_xfered;
 774                 mdata->xfer_len = min(MTK_SPI_MAX_FIFO_SIZE, len);
 775                 mtk_spi_setup_packet(host);
 776
 777                 if (xfer->tx_buf) {
 778                         cnt = mdata->xfer_len / 4;
 779                         iowrite32_rep(mdata->base + SPI_TX_DATA_REG,
 780                                         xfer->tx_buf + mdata->num_xfered, cnt);
 781
 782                         remainder = mdata->xfer_len % 4;
 783                         if (remainder > 0) {
 784                                 reg_val = 0;
 785                                 memcpy(&reg_val,
 786                                         xfer->tx_buf + (cnt * 4) + mdata->num_xfered,
 787                                         remainder);
 788                                 writel(reg_val, mdata->base + SPI_TX_DATA_REG);
 789                         }
 790                 }
 791
 792                 mtk_spi_enable_transfer(host);
 793
 794                 return IRQ_HANDLED;
 795         }
 796
 797         if (mdata->tx_sgl)
 798                 xfer->tx_dma += mdata->xfer_len;
 799         if (mdata->rx_sgl)
 800                 xfer->rx_dma += mdata->xfer_len;
 801
 802         if (mdata->tx_sgl && (mdata->tx_sgl_len == 0)) {
 803                 mdata->tx_sgl = sg_next(mdata->tx_sgl);
 804                 if (mdata->tx_sgl) {
 805                         xfer->tx_dma = sg_dma_address(mdata->tx_sgl);
 806                         mdata->tx_sgl_len = sg_dma_len(mdata->tx_sgl);
 807                 }
 808         }
 809         if (mdata->rx_sgl && (mdata->rx_sgl_len == 0)) {
 810                 mdata->rx_sgl = sg_next(mdata->rx_sgl);
 811                 if (mdata->rx_sgl) {
 812                         xfer->rx_dma = sg_dma_address(mdata->rx_sgl);
 813                         mdata->rx_sgl_len = sg_dma_len(mdata->rx_sgl);
 814                 }
 815         }
 816
 817         if (!mdata->tx_sgl && !mdata->rx_sgl) {
 818                 /* spi disable dma */
 819                 cmd = readl(mdata->base + SPI_CMD_REG);
 820                 cmd &= ~SPI_CMD_TX_DMA;
 821                 cmd &= ~SPI_CMD_RX_DMA;
 822                 writel(cmd, mdata->base + SPI_CMD_REG);
 823
 824                 spi_finalize_current_transfer(host);
 825                 return IRQ_HANDLED;
 826         }
 827
 828         mtk_spi_update_mdata_len(host);
 829         mtk_spi_setup_packet(host);
 830         mtk_spi_setup_dma_addr(host, xfer);
 831         mtk_spi_enable_transfer(host);
 832
 833         return IRQ_HANDLED;
 834 }
 835
 836 static irqreturn_t mtk_spi_interrupt(int irq, void *dev_id)
 837 {
 838         struct spi_controller *host = dev_id;
 839         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 840         u32 reg_val;
 841
 842         reg_val = readl(mdata->base + SPI_STATUS0_REG);
 843         if (reg_val & MTK_SPI_PAUSE_INT_STATUS)
 844                 mdata->state = MTK_SPI_PAUSED;
 845         else
 846                 mdata->state = MTK_SPI_IDLE;
 847
 848         /* SPI-MEM ops */
 849         if (mdata->use_spimem) {
 850                 complete(&mdata->spimem_done);
 851                 return IRQ_HANDLED;
 852         }
 853
 854         return IRQ_WAKE_THREAD;
 855 }
 856
 857 static int mtk_spi_mem_adjust_op_size(struct spi_mem *mem,
 858                                       struct spi_mem_op *op)
 859 {
 860         int opcode_len;
 861
 862         if (op->data.dir != SPI_MEM_NO_DATA) {
 863                 opcode_len = 1 + op->addr.nbytes + op->dummy.nbytes;
 864                 if (opcode_len + op->data.nbytes > MTK_SPI_IPM_PACKET_SIZE) {
 865                         op->data.nbytes = MTK_SPI_IPM_PACKET_SIZE - opcode_len;
 866                         /* force data buffer dma-aligned. */
 867                         op->data.nbytes -= op->data.nbytes % 4;
 868                 }
 869         }
 870
 871         return 0;
 872 }
 873
 874 static bool mtk_spi_mem_supports_op(struct spi_mem *mem,
 875                                     const struct spi_mem_op *op)
 876 {
 877         if (!spi_mem_default_supports_op(mem, op))
 878                 return false;
 879
 880         if (op->addr.nbytes && op->dummy.nbytes &&
 881             op->addr.buswidth != op->dummy.buswidth)
 882                 return false;
 883
 884         if (op->addr.nbytes + op->dummy.nbytes > 16)
 885                 return false;
 886
 887         if (op->data.nbytes > MTK_SPI_IPM_PACKET_SIZE) {
 888                 if (op->data.nbytes / MTK_SPI_IPM_PACKET_SIZE >
 889                     MTK_SPI_IPM_PACKET_LOOP ||
 890                     op->data.nbytes % MTK_SPI_IPM_PACKET_SIZE != 0)
 891                         return false;
 892         }
 893
 894         return true;
 895 }
 896
 897 static void mtk_spi_mem_setup_dma_xfer(struct spi_controller *host,
 898                                        const struct spi_mem_op *op)
 899 {
 900         struct mtk_spi *mdata = spi_controller_get_devdata(host);
 901
 902         writel((u32)(mdata->tx_dma & MTK_SPI_32BITS_MASK),
 903                mdata->base + SPI_TX_SRC_REG);
 904 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 905         if (mdata->dev_comp->dma_ext)
 906                 writel((u32)(mdata->tx_dma >> 32),
 907                        mdata->base + SPI_TX_SRC_REG_64);
 908 #endif
 909
 910         if (op->data.dir == SPI_MEM_DATA_IN) {
 911                 writel((u32)(mdata->rx_dma & MTK_SPI_32BITS_MASK),
 912                        mdata->base + SPI_RX_DST_REG);
 913 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 914                 if (mdata->dev_comp->dma_ext)
 915                         writel((u32)(mdata->rx_dma >> 32),
 916                                mdata->base + SPI_RX_DST_REG_64);
 917 #endif
 918         }
 919 }
 920
 921 static int mtk_spi_transfer_wait(struct spi_mem *mem,
 922                                  const struct spi_mem_op *op)
 923 {
 924         struct mtk_spi *mdata = spi_controller_get_devdata(mem->spi->controller);
 925         /*
 926          * For each byte we wait for 8 cycles of the SPI clock.
 927          * Since speed is defined in Hz and we want milliseconds,
 928          * so it should be 8 * 1000.
 929          */
 930         u64 ms = 8000LL;
 931
 932         if (op->data.dir == SPI_MEM_NO_DATA)
 933                 ms *= 32; /* prevent we may get 0 for short transfers. */
 934         else
 935                 ms *= op->data.nbytes;
 936         ms = div_u64(ms, mem->spi->max_speed_hz);
 937         ms += ms + 1000; /* 1s tolerance */
 938
 939         if (ms > UINT_MAX)
 940                 ms = UINT_MAX;
 941
 942         if (!wait_for_completion_timeout(&mdata->spimem_done,
 943                                          msecs_to_jiffies(ms))) {
 944                 dev_err(mdata->dev, "spi-mem transfer timeout\n");
 945                 return -ETIMEDOUT;
 946         }
 947
 948         return 0;
 949 }
 950
 951 static int mtk_spi_mem_exec_op(struct spi_mem *mem,
 952                                const struct spi_mem_op *op)
 953 {
 954         struct mtk_spi *mdata = spi_controller_get_devdata(mem->spi->controller);
 955         u32 reg_val, nio, tx_size;
 956         char *tx_tmp_buf, *rx_tmp_buf;
 957         int ret = 0;
 958
 959         mdata->use_spimem = true;
 960         reinit_completion(&mdata->spimem_done);
 961
 962         mtk_spi_reset(mdata);
 963         mtk_spi_hw_init(mem->spi->controller, mem->spi);
 964         mtk_spi_prepare_transfer(mem->spi->controller, mem->spi->max_speed_hz);
 965
 966         reg_val = readl(mdata->base + SPI_CFG3_IPM_REG);
 967         /* opcode byte len */
 968         reg_val &= ~SPI_CFG3_IPM_CMD_BYTELEN_MASK;
 969         reg_val |= 1 << SPI_CFG3_IPM_CMD_BYTELEN_OFFSET;
 970
 971         /* addr & dummy byte len */
 972         reg_val &= ~SPI_CFG3_IPM_ADDR_BYTELEN_MASK;
 973         if (op->addr.nbytes || op->dummy.nbytes)
 974                 reg_val |= (op->addr.nbytes + op->dummy.nbytes) <<
 975                             SPI_CFG3_IPM_ADDR_BYTELEN_OFFSET;
 976
 977         /* data byte len */
 978         if (op->data.dir == SPI_MEM_NO_DATA) {
 979                 reg_val |= SPI_CFG3_IPM_NODATA_FLAG;
 980                 writel(0, mdata->base + SPI_CFG1_REG);
 981         } else {
 982                 reg_val &= ~SPI_CFG3_IPM_NODATA_FLAG;
 983                 mdata->xfer_len = op->data.nbytes;
 984                 mtk_spi_setup_packet(mem->spi->controller);
 985         }
 986
 987         if (op->addr.nbytes || op->dummy.nbytes) {
 988                 if (op->addr.buswidth == 1 || op->dummy.buswidth == 1)
 989                         reg_val |= SPI_CFG3_IPM_XMODE_EN;
 990                 else
 991                         reg_val &= ~SPI_CFG3_IPM_XMODE_EN;
 992         }
 993
 994         if (op->addr.buswidth == 2 ||
 995             op->dummy.buswidth == 2 ||
 996             op->data.buswidth == 2)
 997                 nio = 2;
 998         else if (op->addr.buswidth == 4 ||
 999                  op->dummy.buswidth == 4 ||
1000                  op->data.buswidth == 4)
1001                 nio = 4;
1002         else
1003                 nio = 1;
1004
1005         reg_val &= ~SPI_CFG3_IPM_CMD_PIN_MODE_MASK;
1006         reg_val |= PIN_MODE_CFG(nio);
1007
1008         reg_val |= SPI_CFG3_IPM_HALF_DUPLEX_EN;
1009         if (op->data.dir == SPI_MEM_DATA_IN)
1010                 reg_val |= SPI_CFG3_IPM_HALF_DUPLEX_DIR;
1011         else
1012                 reg_val &= ~SPI_CFG3_IPM_HALF_DUPLEX_DIR;
1013         writel(reg_val, mdata->base + SPI_CFG3_IPM_REG);
1014
1015         tx_size = 1 + op->addr.nbytes + op->dummy.nbytes;
1016         if (op->data.dir == SPI_MEM_DATA_OUT)
1017                 tx_size += op->data.nbytes;
1018
1019         tx_size = max_t(u32, tx_size, 32);
1020
1021         tx_tmp_buf = kzalloc(tx_size, GFP_KERNEL | GFP_DMA);
1022         if (!tx_tmp_buf) {
1023                 mdata->use_spimem = false;
1024                 return -ENOMEM;
1025         }
1026
1027         tx_tmp_buf[0] = op->cmd.opcode;
1028
1029         if (op->addr.nbytes) {
1030                 int i;
1031
1032                 for (i = 0; i < op->addr.nbytes; i++)
1033                         tx_tmp_buf[i + 1] = op->addr.val >>
1034                                         (8 * (op->addr.nbytes - i - 1));
1035         }
1036
1037         if (op->dummy.nbytes)
1038                 memset(tx_tmp_buf + op->addr.nbytes + 1,
1039                        0xff,
1040                        op->dummy.nbytes);
1041
1042         if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT)
1043                 memcpy(tx_tmp_buf + op->dummy.nbytes + op->addr.nbytes + 1,
1044                        op->data.buf.out,
1045                        op->data.nbytes);
1046
1047         mdata->tx_dma = dma_map_single(mdata->dev, tx_tmp_buf,
1048                                        tx_size, DMA_TO_DEVICE);
1049         if (dma_mapping_error(mdata->dev, mdata->tx_dma)) {
1050                 ret = -ENOMEM;
1051                 goto err_exit;
1052         }
1053
1054         if (op->data.dir == SPI_MEM_DATA_IN) {
1055                 if (!IS_ALIGNED((size_t)op->data.buf.in, 4)) {
1056                         rx_tmp_buf = kzalloc(op->data.nbytes,
1057                                              GFP_KERNEL | GFP_DMA);
1058                         if (!rx_tmp_buf) {
1059                                 ret = -ENOMEM;
1060                                 goto unmap_tx_dma;
1061                         }
1062                 } else {
1063                         rx_tmp_buf = op->data.buf.in;
1064                 }
1065
1066                 mdata->rx_dma = dma_map_single(mdata->dev,
1067                                                rx_tmp_buf,
1068                                                op->data.nbytes,
1069                                                DMA_FROM_DEVICE);
1070                 if (dma_mapping_error(mdata->dev, mdata->rx_dma)) {
1071                         ret = -ENOMEM;
1072                         goto kfree_rx_tmp_buf;
1073                 }
1074         }
1075
1076         reg_val = readl(mdata->base + SPI_CMD_REG);
1077         reg_val |= SPI_CMD_TX_DMA;
1078         if (op->data.dir == SPI_MEM_DATA_IN)
1079                 reg_val |= SPI_CMD_RX_DMA;
1080         writel(reg_val, mdata->base + SPI_CMD_REG);
1081
1082         mtk_spi_mem_setup_dma_xfer(mem->spi->controller, op);
1083
1084         mtk_spi_enable_transfer(mem->spi->controller);
1085
1086         /* Wait for the interrupt. */
1087         ret = mtk_spi_transfer_wait(mem, op);
1088         if (ret)
1089                 goto unmap_rx_dma;
1090
1091         /* spi disable dma */
1092         reg_val = readl(mdata->base + SPI_CMD_REG);
1093         reg_val &= ~SPI_CMD_TX_DMA;
1094         if (op->data.dir == SPI_MEM_DATA_IN)
1095                 reg_val &= ~SPI_CMD_RX_DMA;
1096         writel(reg_val, mdata->base + SPI_CMD_REG);
1097
1098 unmap_rx_dma:
1099         if (op->data.dir == SPI_MEM_DATA_IN) {
1100                 dma_unmap_single(mdata->dev, mdata->rx_dma,
1101                                  op->data.nbytes, DMA_FROM_DEVICE);
1102                 if (!IS_ALIGNED((size_t)op->data.buf.in, 4))
1103                         memcpy(op->data.buf.in, rx_tmp_buf, op->data.nbytes);
1104         }
1105 kfree_rx_tmp_buf:
1106         if (op->data.dir == SPI_MEM_DATA_IN &&
1107             !IS_ALIGNED((size_t)op->data.buf.in, 4))
1108                 kfree(rx_tmp_buf);
1109 unmap_tx_dma:
1110         dma_unmap_single(mdata->dev, mdata->tx_dma,
1111                          tx_size, DMA_TO_DEVICE);
1112 err_exit:
1113         kfree(tx_tmp_buf);
1114         mdata->use_spimem = false;
1115
1116         return ret;
1117 }
1118
1119 static const struct spi_controller_mem_ops mtk_spi_mem_ops = {
1120         .adjust_op_size = mtk_spi_mem_adjust_op_size,
1121         .supports_op = mtk_spi_mem_supports_op,
1122         .exec_op = mtk_spi_mem_exec_op,
1123 };
1124
1125 static int mtk_spi_probe(struct platform_device *pdev)
1126 {
1127         struct device *dev = &pdev->dev;
1128         struct spi_controller *host;
1129         struct mtk_spi *mdata;
1130         int i, irq, ret, addr_bits;
1131
1132         host = devm_spi_alloc_host(dev, sizeof(*mdata));
1133         if (!host)
1134                 return dev_err_probe(dev, -ENOMEM, "failed to alloc spi host\n");
1135
1136         host->auto_runtime_pm = true;
1137         host->dev.of_node = dev->of_node;
1138         host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LSB_FIRST;
1139
1140         host->set_cs = mtk_spi_set_cs;
1141         host->prepare_message = mtk_spi_prepare_message;
1142         host->transfer_one = mtk_spi_transfer_one;
1143         host->can_dma = mtk_spi_can_dma;
1144         host->setup = mtk_spi_setup;
1145         host->set_cs_timing = mtk_spi_set_hw_cs_timing;
1146         host->use_gpio_descriptors = true;
1147
1148         mdata = spi_controller_get_devdata(host);
1149         mdata->dev_comp = device_get_match_data(dev);
1150
1151         if (mdata->dev_comp->enhance_timing)
1152                 host->mode_bits |= SPI_CS_HIGH;
1153
1154         if (mdata->dev_comp->must_tx)
1155                 host->flags = SPI_CONTROLLER_MUST_TX;
1156         if (mdata->dev_comp->ipm_design)
1157                 host->mode_bits |= SPI_LOOP | SPI_RX_DUAL | SPI_TX_DUAL |
1158                                    SPI_RX_QUAD | SPI_TX_QUAD;
1159
1160         if (mdata->dev_comp->ipm_design) {
1161                 mdata->dev = dev;
1162                 host->mem_ops = &mtk_spi_mem_ops;
1163                 init_completion(&mdata->spimem_done);
1164         }
1165
1166         if (mdata->dev_comp->need_pad_sel) {
1167                 mdata->pad_num = of_property_count_u32_elems(dev->of_node,
1168                         "mediatek,pad-select");
1169                 if (mdata->pad_num < 0)
1170                         return dev_err_probe(dev, -EINVAL,
1171                                 "No 'mediatek,pad-select' property\n");
1172
1173                 mdata->pad_sel = devm_kmalloc_array(dev, mdata->pad_num,
1174                                                     sizeof(u32), GFP_KERNEL);
1175                 if (!mdata->pad_sel)
1176                         return -ENOMEM;
1177
1178                 for (i = 0; i < mdata->pad_num; i++) {
1179                         of_property_read_u32_index(dev->of_node,
1180                                                    "mediatek,pad-select",
1181                                                    i, &mdata->pad_sel[i]);
1182                         if (mdata->pad_sel[i] > MT8173_SPI_MAX_PAD_SEL)
1183                                 return dev_err_probe(dev, -EINVAL,
1184                                                      "wrong pad-sel[%d]: %u\n",
1185                                                      i, mdata->pad_sel[i]);
1186                 }
1187         }
1188
1189         platform_set_drvdata(pdev, host);
1190         mdata->base = devm_platform_ioremap_resource(pdev, 0);
1191         if (IS_ERR(mdata->base))
1192                 return PTR_ERR(mdata->base);
1193
1194         irq = platform_get_irq(pdev, 0);
1195         if (irq < 0)
1196                 return irq;
1197
1198         if (!dev->dma_mask)
1199                 dev->dma_mask = &dev->coherent_dma_mask;
1200
1201         if (mdata->dev_comp->ipm_design)
1202                 dma_set_max_seg_size(dev, SZ_16M);
1203         else
1204                 dma_set_max_seg_size(dev, SZ_256K);
1205
1206         mdata->parent_clk = devm_clk_get(dev, "parent-clk");
1207         if (IS_ERR(mdata->parent_clk))
1208                 return dev_err_probe(dev, PTR_ERR(mdata->parent_clk),
1209                                      "failed to get parent-clk\n");
1210
1211         mdata->sel_clk = devm_clk_get(dev, "sel-clk");
1212         if (IS_ERR(mdata->sel_clk))
1213                 return dev_err_probe(dev, PTR_ERR(mdata->sel_clk), "failed to get sel-clk\n");
1214
1215         mdata->spi_clk = devm_clk_get(dev, "spi-clk");
1216         if (IS_ERR(mdata->spi_clk))
1217                 return dev_err_probe(dev, PTR_ERR(mdata->spi_clk), "failed to get spi-clk\n");
1218
1219         mdata->spi_hclk = devm_clk_get_optional(dev, "hclk");
1220         if (IS_ERR(mdata->spi_hclk))
1221                 return dev_err_probe(dev, PTR_ERR(mdata->spi_hclk), "failed to get hclk\n");
1222
1223         ret = clk_set_parent(mdata->sel_clk, mdata->parent_clk);
1224         if (ret < 0)
1225                 return dev_err_probe(dev, ret, "failed to clk_set_parent\n");
1226
1227         ret = clk_prepare_enable(mdata->spi_hclk);
1228         if (ret < 0)
1229                 return dev_err_probe(dev, ret, "failed to enable hclk\n");
1230
1231         ret = clk_prepare_enable(mdata->spi_clk);
1232         if (ret < 0) {
1233                 clk_disable_unprepare(mdata->spi_hclk);
1234                 return dev_err_probe(dev, ret, "failed to enable spi_clk\n");
1235         }
1236
1237         mdata->spi_clk_hz = clk_get_rate(mdata->spi_clk);
1238
1239         if (mdata->dev_comp->no_need_unprepare) {
1240                 clk_disable(mdata->spi_clk);
1241                 clk_disable(mdata->spi_hclk);
1242         } else {
1243                 clk_disable_unprepare(mdata->spi_clk);
1244                 clk_disable_unprepare(mdata->spi_hclk);
1245         }
1246
1247         if (mdata->dev_comp->need_pad_sel) {
1248                 if (mdata->pad_num != host->num_chipselect)
1249                         return dev_err_probe(dev, -EINVAL,
1250                                 "pad_num does not match num_chipselect(%d != %d)\n",
1251                                 mdata->pad_num, host->num_chipselect);
1252
1253                 if (!host->cs_gpiods && host->num_chipselect > 1)
1254                         return dev_err_probe(dev, -EINVAL,
1255                                 "cs_gpios not specified and num_chipselect > 1\n");
1256         }
1257
1258         if (mdata->dev_comp->dma_ext)
1259                 addr_bits = DMA_ADDR_EXT_BITS;
1260         else
1261                 addr_bits = DMA_ADDR_DEF_BITS;
1262         ret = dma_set_mask(dev, DMA_BIT_MASK(addr_bits));
1263         if (ret)
1264                 dev_notice(dev, "SPI dma_set_mask(%d) failed, ret:%d\n",
1265                            addr_bits, ret);
1266
1267         ret = devm_request_threaded_irq(dev, irq, mtk_spi_interrupt,
1268                                         mtk_spi_interrupt_thread,
1269                                         IRQF_TRIGGER_NONE, dev_name(dev), host);
1270         if (ret)
1271                 return dev_err_probe(dev, ret, "failed to register irq\n");
1272
1273         pm_runtime_enable(dev);
1274
1275         ret = devm_spi_register_controller(dev, host);
1276         if (ret) {
1277                 pm_runtime_disable(dev);
1278                 return dev_err_probe(dev, ret, "failed to register host\n");
1279         }
1280
1281         return 0;
1282 }
1283
1284 static void mtk_spi_remove(struct platform_device *pdev)
1285 {
1286         struct spi_controller *host = platform_get_drvdata(pdev);
1287         struct mtk_spi *mdata = spi_controller_get_devdata(host);
1288         int ret;
1289
1290         if (mdata->use_spimem && !completion_done(&mdata->spimem_done))
1291                 complete(&mdata->spimem_done);
1292
1293         ret = pm_runtime_get_sync(&pdev->dev);
1294         if (ret < 0) {
1295                 dev_warn(&pdev->dev, "Failed to resume hardware (%pe)\n", ERR_PTR(ret));
1296         } else {
1297                 /*
1298                  * If pm runtime resume failed, clks are disabled and
1299                  * unprepared. So don't access the hardware and skip clk
1300                  * unpreparing.
1301                  */
1302                 mtk_spi_reset(mdata);
1303
1304                 if (mdata->dev_comp->no_need_unprepare) {
1305                         clk_unprepare(mdata->spi_clk);
1306                         clk_unprepare(mdata->spi_hclk);
1307                 }
1308         }
1309
1310         pm_runtime_put_noidle(&pdev->dev);
1311         pm_runtime_disable(&pdev->dev);
1312 }
1313
1314 #ifdef CONFIG_PM_SLEEP
1315 static int mtk_spi_suspend(struct device *dev)
1316 {
1317         int ret;
1318         struct spi_controller *host = dev_get_drvdata(dev);
1319         struct mtk_spi *mdata = spi_controller_get_devdata(host);
1320
1321         ret = spi_controller_suspend(host);
1322         if (ret)
1323                 return ret;
1324
1325         if (!pm_runtime_suspended(dev)) {
1326                 clk_disable_unprepare(mdata->spi_clk);
1327                 clk_disable_unprepare(mdata->spi_hclk);
1328         }
1329
1330         pinctrl_pm_select_sleep_state(dev);
1331
1332         return 0;
1333 }
1334
1335 static int mtk_spi_resume(struct device *dev)
1336 {
1337         int ret;
1338         struct spi_controller *host = dev_get_drvdata(dev);
1339         struct mtk_spi *mdata = spi_controller_get_devdata(host);
1340
1341         pinctrl_pm_select_default_state(dev);
1342
1343         if (!pm_runtime_suspended(dev)) {
1344                 ret = clk_prepare_enable(mdata->spi_clk);
1345                 if (ret < 0) {
1346                         dev_err(dev, "failed to enable spi_clk (%d)\n", ret);
1347                         return ret;
1348                 }
1349
1350                 ret = clk_prepare_enable(mdata->spi_hclk);
1351                 if (ret < 0) {
1352                         dev_err(dev, "failed to enable spi_hclk (%d)\n", ret);
1353                         clk_disable_unprepare(mdata->spi_clk);
1354                         return ret;
1355                 }
1356         }
1357
1358         ret = spi_controller_resume(host);
1359         if (ret < 0) {
1360                 clk_disable_unprepare(mdata->spi_clk);
1361                 clk_disable_unprepare(mdata->spi_hclk);
1362         }
1363
1364         return ret;
1365 }
1366 #endif /* CONFIG_PM_SLEEP */
1367
1368 #ifdef CONFIG_PM
1369 static int mtk_spi_runtime_suspend(struct device *dev)
1370 {
1371         struct spi_controller *host = dev_get_drvdata(dev);
1372         struct mtk_spi *mdata = spi_controller_get_devdata(host);
1373
1374         if (mdata->dev_comp->no_need_unprepare) {
1375                 clk_disable(mdata->spi_clk);
1376                 clk_disable(mdata->spi_hclk);
1377         } else {
1378                 clk_disable_unprepare(mdata->spi_clk);
1379                 clk_disable_unprepare(mdata->spi_hclk);
1380         }
1381
1382         return 0;
1383 }
1384
1385 static int mtk_spi_runtime_resume(struct device *dev)
1386 {
1387         struct spi_controller *host = dev_get_drvdata(dev);
1388         struct mtk_spi *mdata = spi_controller_get_devdata(host);
1389         int ret;
1390
1391         if (mdata->dev_comp->no_need_unprepare) {
1392                 ret = clk_enable(mdata->spi_clk);
1393                 if (ret < 0) {
1394                         dev_err(dev, "failed to enable spi_clk (%d)\n", ret);
1395                         return ret;
1396                 }
1397                 ret = clk_enable(mdata->spi_hclk);
1398                 if (ret < 0) {
1399                         dev_err(dev, "failed to enable spi_hclk (%d)\n", ret);
1400                         clk_disable(mdata->spi_clk);
1401                         return ret;
1402                 }
1403         } else {
1404                 ret = clk_prepare_enable(mdata->spi_clk);
1405                 if (ret < 0) {
1406                         dev_err(dev, "failed to prepare_enable spi_clk (%d)\n", ret);
1407                         return ret;
1408                 }
1409
1410                 ret = clk_prepare_enable(mdata->spi_hclk);
1411                 if (ret < 0) {
1412                         dev_err(dev, "failed to prepare_enable spi_hclk (%d)\n", ret);
1413                         clk_disable_unprepare(mdata->spi_clk);
1414                         return ret;
1415                 }
1416         }
1417
1418         return 0;
1419 }
1420 #endif /* CONFIG_PM */
1421
1422 static const struct dev_pm_ops mtk_spi_pm = {
1423         SET_SYSTEM_SLEEP_PM_OPS(mtk_spi_suspend, mtk_spi_resume)
1424         SET_RUNTIME_PM_OPS(mtk_spi_runtime_suspend,
1425                            mtk_spi_runtime_resume, NULL)
1426 };
1427
1428 static struct platform_driver mtk_spi_driver = {
1429         .driver = {
1430                 .name = "mtk-spi",
1431                 .pm     = &mtk_spi_pm,
1432                 .of_match_table = mtk_spi_of_match,
1433         },
1434         .probe = mtk_spi_probe,
1435         .remove = mtk_spi_remove,
1436 };
1437
1438 module_platform_driver(mtk_spi_driver);
1439
1440 MODULE_DESCRIPTION("MTK SPI Controller driver");
1441 MODULE_AUTHOR("Leilk Liu <[email protected]>");
1442 MODULE_LICENSE("GPL v2");
1443 MODULE_ALIAS("platform:mtk-spi");