drivers/spi/spi-omap2-mcspi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * OMAP2 McSPI controller driver
   4  *
   5  * Copyright (C) 2005, 2006 Nokia Corporation
   6  * Author:      Samuel Ortiz <[email protected]> and
   7  *              Juha Yrjola <[email protected]>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/device.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/err.h>
  20 #include <linux/clk.h>
  21 #include <linux/io.h>
  22 #include <linux/slab.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/of.h>
  25 #include <linux/of_device.h>
  26 #include <linux/gcd.h>
  27
  28 #include <linux/spi/spi.h>
  29
  30 #include "internals.h"
  31
  32 #include <linux/platform_data/spi-omap2-mcspi.h>
  33
  34 #define OMAP2_MCSPI_MAX_FREQ            48000000
  35 #define OMAP2_MCSPI_MAX_DIVIDER         4096
  36 #define OMAP2_MCSPI_MAX_FIFODEPTH       64
  37 #define OMAP2_MCSPI_MAX_FIFOWCNT        0xFFFF
  38 #define SPI_AUTOSUSPEND_TIMEOUT         2000
  39
  40 #define OMAP2_MCSPI_REVISION            0x00
  41 #define OMAP2_MCSPI_SYSSTATUS           0x14
  42 #define OMAP2_MCSPI_IRQSTATUS           0x18
  43 #define OMAP2_MCSPI_IRQENABLE           0x1c
  44 #define OMAP2_MCSPI_WAKEUPENABLE        0x20
  45 #define OMAP2_MCSPI_SYST                0x24
  46 #define OMAP2_MCSPI_MODULCTRL           0x28
  47 #define OMAP2_MCSPI_XFERLEVEL           0x7c
  48
  49 /* per-channel banks, 0x14 bytes each, first is: */
  50 #define OMAP2_MCSPI_CHCONF0             0x2c
  51 #define OMAP2_MCSPI_CHSTAT0             0x30
  52 #define OMAP2_MCSPI_CHCTRL0             0x34
  53 #define OMAP2_MCSPI_TX0                 0x38
  54 #define OMAP2_MCSPI_RX0                 0x3c
  55
  56 /* per-register bitmasks: */
  57 #define OMAP2_MCSPI_IRQSTATUS_EOW       BIT(17)
  58
  59 #define OMAP2_MCSPI_MODULCTRL_SINGLE    BIT(0)
  60 #define OMAP2_MCSPI_MODULCTRL_MS        BIT(2)
  61 #define OMAP2_MCSPI_MODULCTRL_STEST     BIT(3)
  62
  63 #define OMAP2_MCSPI_CHCONF_PHA          BIT(0)
  64 #define OMAP2_MCSPI_CHCONF_POL          BIT(1)
  65 #define OMAP2_MCSPI_CHCONF_CLKD_MASK    (0x0f << 2)
  66 #define OMAP2_MCSPI_CHCONF_EPOL         BIT(6)
  67 #define OMAP2_MCSPI_CHCONF_WL_MASK      (0x1f << 7)
  68 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY  BIT(12)
  69 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY  BIT(13)
  70 #define OMAP2_MCSPI_CHCONF_TRM_MASK     (0x03 << 12)
  71 #define OMAP2_MCSPI_CHCONF_DMAW         BIT(14)
  72 #define OMAP2_MCSPI_CHCONF_DMAR         BIT(15)
  73 #define OMAP2_MCSPI_CHCONF_DPE0         BIT(16)
  74 #define OMAP2_MCSPI_CHCONF_DPE1         BIT(17)
  75 #define OMAP2_MCSPI_CHCONF_IS           BIT(18)
  76 #define OMAP2_MCSPI_CHCONF_TURBO        BIT(19)
  77 #define OMAP2_MCSPI_CHCONF_FORCE        BIT(20)
  78 #define OMAP2_MCSPI_CHCONF_FFET         BIT(27)
  79 #define OMAP2_MCSPI_CHCONF_FFER         BIT(28)
  80 #define OMAP2_MCSPI_CHCONF_CLKG         BIT(29)
  81
  82 #define OMAP2_MCSPI_CHSTAT_RXS          BIT(0)
  83 #define OMAP2_MCSPI_CHSTAT_TXS          BIT(1)
  84 #define OMAP2_MCSPI_CHSTAT_EOT          BIT(2)
  85 #define OMAP2_MCSPI_CHSTAT_TXFFE        BIT(3)
  86
  87 #define OMAP2_MCSPI_CHCTRL_EN           BIT(0)
  88 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK  (0xff << 8)
  89
  90 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN   BIT(0)
  91
  92 /* We have 2 DMA channels per CS, one for RX and one for TX */
  93 struct omap2_mcspi_dma {
  94         struct dma_chan *dma_tx;
  95         struct dma_chan *dma_rx;
  96
  97         struct completion dma_tx_completion;
  98         struct completion dma_rx_completion;
  99
 100         char dma_rx_ch_name[14];
 101         char dma_tx_ch_name[14];
 102 };
 103
 104 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 105  * cache operations; better heuristics consider wordsize and bitrate.
 106  */
 107 #define DMA_MIN_BYTES                   160
 108
 109
 110 /*
 111  * Used for context save and restore, structure members to be updated whenever
 112  * corresponding registers are modified.
 113  */
 114 struct omap2_mcspi_regs {
 115         u32 modulctrl;
 116         u32 wakeupenable;
 117         struct list_head cs;
 118 };
 119
 120 struct omap2_mcspi {
 121         struct completion       txdone;
 122         struct spi_controller   *ctlr;
 123         /* Virtual base address of the controller */
 124         void __iomem            *base;
 125         unsigned long           phys;
 126         /* SPI1 has 4 channels, while SPI2 has 2 */
 127         struct omap2_mcspi_dma  *dma_channels;
 128         struct device           *dev;
 129         struct omap2_mcspi_regs ctx;
 130         struct clk              *ref_clk;
 131         int                     fifo_depth;
 132         bool                    target_aborted;
 133         unsigned int            pin_dir:1;
 134         size_t                  max_xfer_len;
 135         u32                     ref_clk_hz;
 136         bool                    use_multi_mode;
 137 };
 138
 139 struct omap2_mcspi_cs {
 140         void __iomem            *base;
 141         unsigned long           phys;
 142         int                     word_len;
 143         u16                     mode;
 144         struct list_head        node;
 145         /* Context save and restore shadow register */
 146         u32                     chconf0, chctrl0;
 147 };
 148
 149 static inline void mcspi_write_reg(struct spi_controller *ctlr,
 150                 int idx, u32 val)
 151 {
 152         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
 153
 154         writel_relaxed(val, mcspi->base + idx);
 155 }
 156
 157 static inline u32 mcspi_read_reg(struct spi_controller *ctlr, int idx)
 158 {
 159         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
 160
 161         return readl_relaxed(mcspi->base + idx);
 162 }
 163
 164 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 165                 int idx, u32 val)
 166 {
 167         struct omap2_mcspi_cs   *cs = spi->controller_state;
 168
 169         writel_relaxed(val, cs->base +  idx);
 170 }
 171
 172 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 173 {
 174         struct omap2_mcspi_cs   *cs = spi->controller_state;
 175
 176         return readl_relaxed(cs->base + idx);
 177 }
 178
 179 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 180 {
 181         struct omap2_mcspi_cs *cs = spi->controller_state;
 182
 183         return cs->chconf0;
 184 }
 185
 186 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 187 {
 188         struct omap2_mcspi_cs *cs = spi->controller_state;
 189
 190         cs->chconf0 = val;
 191         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 192         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 193 }
 194
 195 static inline int mcspi_bytes_per_word(int word_len)
 196 {
 197         if (word_len <= 8)
 198                 return 1;
 199         else if (word_len <= 16)
 200                 return 2;
 201         else /* word_len <= 32 */
 202                 return 4;
 203 }
 204
 205 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 206                 int is_read, int enable)
 207 {
 208         u32 l, rw;
 209
 210         l = mcspi_cached_chconf0(spi);
 211
 212         if (is_read) /* 1 is read, 0 write */
 213                 rw = OMAP2_MCSPI_CHCONF_DMAR;
 214         else
 215                 rw = OMAP2_MCSPI_CHCONF_DMAW;
 216
 217         if (enable)
 218                 l |= rw;
 219         else
 220                 l &= ~rw;
 221
 222         mcspi_write_chconf0(spi, l);
 223 }
 224
 225 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 226 {
 227         struct omap2_mcspi_cs *cs = spi->controller_state;
 228         u32 l;
 229
 230         l = cs->chctrl0;
 231         if (enable)
 232                 l |= OMAP2_MCSPI_CHCTRL_EN;
 233         else
 234                 l &= ~OMAP2_MCSPI_CHCTRL_EN;
 235         cs->chctrl0 = l;
 236         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 237         /* Flash post-writes */
 238         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 239 }
 240
 241 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 242 {
 243         struct omap2_mcspi *mcspi = spi_controller_get_devdata(spi->controller);
 244         u32 l;
 245
 246         /* The controller handles the inverted chip selects
 247          * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 248          * the inversion from the core spi_set_cs function.
 249          */
 250         if (spi->mode & SPI_CS_HIGH)
 251                 enable = !enable;
 252
 253         if (spi->controller_state) {
 254                 int err = pm_runtime_resume_and_get(mcspi->dev);
 255                 if (err < 0) {
 256                         dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 257                         return;
 258                 }
 259
 260                 l = mcspi_cached_chconf0(spi);
 261
 262                 /* Only enable chip select manually if single mode is used */
 263                 if (mcspi->use_multi_mode) {
 264                         l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 265                 } else {
 266                         if (enable)
 267                                 l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 268                         else
 269                                 l |= OMAP2_MCSPI_CHCONF_FORCE;
 270                 }
 271
 272                 mcspi_write_chconf0(spi, l);
 273
 274                 pm_runtime_mark_last_busy(mcspi->dev);
 275                 pm_runtime_put_autosuspend(mcspi->dev);
 276         }
 277 }
 278
 279 static void omap2_mcspi_set_mode(struct spi_controller *ctlr)
 280 {
 281         struct omap2_mcspi      *mcspi = spi_controller_get_devdata(ctlr);
 282         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
 283         u32 l;
 284
 285         /*
 286          * Choose host or target mode
 287          */
 288         l = mcspi_read_reg(ctlr, OMAP2_MCSPI_MODULCTRL);
 289         l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
 290         if (spi_controller_is_target(ctlr)) {
 291                 l |= (OMAP2_MCSPI_MODULCTRL_MS);
 292         } else {
 293                 l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
 294
 295                 /* Enable single mode if needed */
 296                 if (mcspi->use_multi_mode)
 297                         l &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
 298                 else
 299                         l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 300         }
 301         mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, l);
 302
 303         ctx->modulctrl = l;
 304 }
 305
 306 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 307                                 struct spi_transfer *t, int enable)
 308 {
 309         struct spi_controller *ctlr = spi->controller;
 310         struct omap2_mcspi_cs *cs = spi->controller_state;
 311         struct omap2_mcspi *mcspi;
 312         unsigned int wcnt;
 313         int max_fifo_depth, bytes_per_word;
 314         u32 chconf, xferlevel;
 315
 316         mcspi = spi_controller_get_devdata(ctlr);
 317
 318         chconf = mcspi_cached_chconf0(spi);
 319         if (enable) {
 320                 bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 321                 if (t->len % bytes_per_word != 0)
 322                         goto disable_fifo;
 323
 324                 if (t->rx_buf != NULL && t->tx_buf != NULL)
 325                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 326                 else
 327                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 328
 329                 wcnt = t->len / bytes_per_word;
 330                 if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 331                         goto disable_fifo;
 332
 333                 xferlevel = wcnt << 16;
 334                 if (t->rx_buf != NULL) {
 335                         chconf |= OMAP2_MCSPI_CHCONF_FFER;
 336                         xferlevel |= (bytes_per_word - 1) << 8;
 337                 }
 338
 339                 if (t->tx_buf != NULL) {
 340                         chconf |= OMAP2_MCSPI_CHCONF_FFET;
 341                         xferlevel |= bytes_per_word - 1;
 342                 }
 343
 344                 mcspi_write_reg(ctlr, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 345                 mcspi_write_chconf0(spi, chconf);
 346                 mcspi->fifo_depth = max_fifo_depth;
 347
 348                 return;
 349         }
 350
 351 disable_fifo:
 352         if (t->rx_buf != NULL)
 353                 chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 354
 355         if (t->tx_buf != NULL)
 356                 chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 357
 358         mcspi_write_chconf0(spi, chconf);
 359         mcspi->fifo_depth = 0;
 360 }
 361
 362 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 363 {
 364         unsigned long timeout;
 365
 366         timeout = jiffies + msecs_to_jiffies(1000);
 367         while (!(readl_relaxed(reg) & bit)) {
 368                 if (time_after(jiffies, timeout)) {
 369                         if (!(readl_relaxed(reg) & bit))
 370                                 return -ETIMEDOUT;
 371                         else
 372                                 return 0;
 373                 }
 374                 cpu_relax();
 375         }
 376         return 0;
 377 }
 378
 379 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
 380                                      struct completion *x)
 381 {
 382         if (spi_controller_is_target(mcspi->ctlr)) {
 383                 if (wait_for_completion_interruptible(x) ||
 384                     mcspi->target_aborted)
 385                         return -EINTR;
 386         } else {
 387                 wait_for_completion(x);
 388         }
 389
 390         return 0;
 391 }
 392
 393 static void omap2_mcspi_rx_callback(void *data)
 394 {
 395         struct spi_device *spi = data;
 396         struct omap2_mcspi *mcspi = spi_controller_get_devdata(spi->controller);
 397         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 398
 399         /* We must disable the DMA RX request */
 400         omap2_mcspi_set_dma_req(spi, 1, 0);
 401
 402         complete(&mcspi_dma->dma_rx_completion);
 403 }
 404
 405 static void omap2_mcspi_tx_callback(void *data)
 406 {
 407         struct spi_device *spi = data;
 408         struct omap2_mcspi *mcspi = spi_controller_get_devdata(spi->controller);
 409         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 410
 411         /* We must disable the DMA TX request */
 412         omap2_mcspi_set_dma_req(spi, 0, 0);
 413
 414         complete(&mcspi_dma->dma_tx_completion);
 415 }
 416
 417 static void omap2_mcspi_tx_dma(struct spi_device *spi,
 418                                 struct spi_transfer *xfer,
 419                                 struct dma_slave_config cfg)
 420 {
 421         struct omap2_mcspi      *mcspi;
 422         struct omap2_mcspi_dma  *mcspi_dma;
 423         struct dma_async_tx_descriptor *tx;
 424
 425         mcspi = spi_controller_get_devdata(spi->controller);
 426         mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 427
 428         dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 429
 430         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 431                                      xfer->tx_sg.nents,
 432                                      DMA_MEM_TO_DEV,
 433                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 434         if (tx) {
 435                 tx->callback = omap2_mcspi_tx_callback;
 436                 tx->callback_param = spi;
 437                 dmaengine_submit(tx);
 438         } else {
 439                 /* FIXME: fall back to PIO? */
 440         }
 441         dma_async_issue_pending(mcspi_dma->dma_tx);
 442         omap2_mcspi_set_dma_req(spi, 0, 1);
 443 }
 444
 445 static unsigned
 446 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 447                                 struct dma_slave_config cfg,
 448                                 unsigned es)
 449 {
 450         struct omap2_mcspi      *mcspi;
 451         struct omap2_mcspi_dma  *mcspi_dma;
 452         unsigned int            count, transfer_reduction = 0;
 453         struct scatterlist      *sg_out[2];
 454         int                     nb_sizes = 0, out_mapped_nents[2], ret, x;
 455         size_t                  sizes[2];
 456         u32                     l;
 457         int                     elements = 0;
 458         int                     word_len, element_count;
 459         struct omap2_mcspi_cs   *cs = spi->controller_state;
 460         void __iomem            *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 461         struct dma_async_tx_descriptor *tx;
 462
 463         mcspi = spi_controller_get_devdata(spi->controller);
 464         mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 465         count = xfer->len;
 466
 467         /*
 468          *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 469          *  it mentions reducing DMA transfer length by one element in host
 470          *  normal mode.
 471          */
 472         if (mcspi->fifo_depth == 0)
 473                 transfer_reduction = es;
 474
 475         word_len = cs->word_len;
 476         l = mcspi_cached_chconf0(spi);
 477
 478         if (word_len <= 8)
 479                 element_count = count;
 480         else if (word_len <= 16)
 481                 element_count = count >> 1;
 482         else /* word_len <= 32 */
 483                 element_count = count >> 2;
 484
 485
 486         dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 487
 488         /*
 489          *  Reduce DMA transfer length by one more if McSPI is
 490          *  configured in turbo mode.
 491          */
 492         if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 493                 transfer_reduction += es;
 494
 495         if (transfer_reduction) {
 496                 /* Split sgl into two. The second sgl won't be used. */
 497                 sizes[0] = count - transfer_reduction;
 498                 sizes[1] = transfer_reduction;
 499                 nb_sizes = 2;
 500         } else {
 501                 /*
 502                  * Don't bother splitting the sgl. This essentially
 503                  * clones the original sgl.
 504                  */
 505                 sizes[0] = count;
 506                 nb_sizes = 1;
 507         }
 508
 509         ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
 510                        sizes, sg_out, out_mapped_nents, GFP_KERNEL);
 511
 512         if (ret < 0) {
 513                 dev_err(&spi->dev, "sg_split failed\n");
 514                 return 0;
 515         }
 516
 517         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
 518                                      out_mapped_nents[0], DMA_DEV_TO_MEM,
 519                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 520         if (tx) {
 521                 tx->callback = omap2_mcspi_rx_callback;
 522                 tx->callback_param = spi;
 523                 dmaengine_submit(tx);
 524         } else {
 525                 /* FIXME: fall back to PIO? */
 526         }
 527
 528         dma_async_issue_pending(mcspi_dma->dma_rx);
 529         omap2_mcspi_set_dma_req(spi, 1, 1);
 530
 531         ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
 532         if (ret || mcspi->target_aborted) {
 533                 dmaengine_terminate_sync(mcspi_dma->dma_rx);
 534                 omap2_mcspi_set_dma_req(spi, 1, 0);
 535                 return 0;
 536         }
 537
 538         for (x = 0; x < nb_sizes; x++)
 539                 kfree(sg_out[x]);
 540
 541         if (mcspi->fifo_depth > 0)
 542                 return count;
 543
 544         /*
 545          *  Due to the DMA transfer length reduction the missing bytes must
 546          *  be read manually to receive all of the expected data.
 547          */
 548         omap2_mcspi_set_enable(spi, 0);
 549
 550         elements = element_count - 1;
 551
 552         if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 553                 elements--;
 554
 555                 if (!mcspi_wait_for_reg_bit(chstat_reg,
 556                                             OMAP2_MCSPI_CHSTAT_RXS)) {
 557                         u32 w;
 558
 559                         w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 560                         if (word_len <= 8)
 561                                 ((u8 *)xfer->rx_buf)[elements++] = w;
 562                         else if (word_len <= 16)
 563                                 ((u16 *)xfer->rx_buf)[elements++] = w;
 564                         else /* word_len <= 32 */
 565                                 ((u32 *)xfer->rx_buf)[elements++] = w;
 566                 } else {
 567                         int bytes_per_word = mcspi_bytes_per_word(word_len);
 568                         dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 569                         count -= (bytes_per_word << 1);
 570                         omap2_mcspi_set_enable(spi, 1);
 571                         return count;
 572                 }
 573         }
 574         if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
 575                 u32 w;
 576
 577                 w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 578                 if (word_len <= 8)
 579                         ((u8 *)xfer->rx_buf)[elements] = w;
 580                 else if (word_len <= 16)
 581                         ((u16 *)xfer->rx_buf)[elements] = w;
 582                 else /* word_len <= 32 */
 583                         ((u32 *)xfer->rx_buf)[elements] = w;
 584         } else {
 585                 dev_err(&spi->dev, "DMA RX last word empty\n");
 586                 count -= mcspi_bytes_per_word(word_len);
 587         }
 588         omap2_mcspi_set_enable(spi, 1);
 589         return count;
 590 }
 591
 592 static unsigned
 593 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 594 {
 595         struct omap2_mcspi      *mcspi;
 596         struct omap2_mcspi_cs   *cs = spi->controller_state;
 597         struct omap2_mcspi_dma  *mcspi_dma;
 598         unsigned int            count;
 599         u8                      *rx;
 600         const u8                *tx;
 601         struct dma_slave_config cfg;
 602         enum dma_slave_buswidth width;
 603         unsigned es;
 604         void __iomem            *chstat_reg;
 605         void __iomem            *irqstat_reg;
 606         int                     wait_res;
 607
 608         mcspi = spi_controller_get_devdata(spi->controller);
 609         mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 610
 611         if (cs->word_len <= 8) {
 612                 width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 613                 es = 1;
 614         } else if (cs->word_len <= 16) {
 615                 width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 616                 es = 2;
 617         } else {
 618                 width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 619                 es = 4;
 620         }
 621
 622         count = xfer->len;
 623
 624         memset(&cfg, 0, sizeof(cfg));
 625         cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 626         cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 627         cfg.src_addr_width = width;
 628         cfg.dst_addr_width = width;
 629         cfg.src_maxburst = 1;
 630         cfg.dst_maxburst = 1;
 631
 632         rx = xfer->rx_buf;
 633         tx = xfer->tx_buf;
 634
 635         mcspi->target_aborted = false;
 636         reinit_completion(&mcspi_dma->dma_tx_completion);
 637         reinit_completion(&mcspi_dma->dma_rx_completion);
 638         reinit_completion(&mcspi->txdone);
 639         if (tx) {
 640                 /* Enable EOW IRQ to know end of tx in target mode */
 641                 if (spi_controller_is_target(spi->controller))
 642                         mcspi_write_reg(spi->controller,
 643                                         OMAP2_MCSPI_IRQENABLE,
 644                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 645                 omap2_mcspi_tx_dma(spi, xfer, cfg);
 646         }
 647
 648         if (rx != NULL)
 649                 count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 650
 651         if (tx != NULL) {
 652                 int ret;
 653
 654                 ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
 655                 if (ret || mcspi->target_aborted) {
 656                         dmaengine_terminate_sync(mcspi_dma->dma_tx);
 657                         omap2_mcspi_set_dma_req(spi, 0, 0);
 658                         return 0;
 659                 }
 660
 661                 if (spi_controller_is_target(mcspi->ctlr)) {
 662                         ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
 663                         if (ret || mcspi->target_aborted)
 664                                 return 0;
 665                 }
 666
 667                 if (mcspi->fifo_depth > 0) {
 668                         irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 669
 670                         if (mcspi_wait_for_reg_bit(irqstat_reg,
 671                                                 OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 672                                 dev_err(&spi->dev, "EOW timed out\n");
 673
 674                         mcspi_write_reg(mcspi->ctlr, OMAP2_MCSPI_IRQSTATUS,
 675                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 676                 }
 677
 678                 /* for TX_ONLY mode, be sure all words have shifted out */
 679                 if (rx == NULL) {
 680                         chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 681                         if (mcspi->fifo_depth > 0) {
 682                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 683                                                 OMAP2_MCSPI_CHSTAT_TXFFE);
 684                                 if (wait_res < 0)
 685                                         dev_err(&spi->dev, "TXFFE timed out\n");
 686                         } else {
 687                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 688                                                 OMAP2_MCSPI_CHSTAT_TXS);
 689                                 if (wait_res < 0)
 690                                         dev_err(&spi->dev, "TXS timed out\n");
 691                         }
 692                         if (wait_res >= 0 &&
 693                                 (mcspi_wait_for_reg_bit(chstat_reg,
 694                                         OMAP2_MCSPI_CHSTAT_EOT) < 0))
 695                                 dev_err(&spi->dev, "EOT timed out\n");
 696                 }
 697         }
 698         return count;
 699 }
 700
 701 static unsigned
 702 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 703 {
 704         struct omap2_mcspi_cs   *cs = spi->controller_state;
 705         unsigned int            count, c;
 706         u32                     l;
 707         void __iomem            *base = cs->base;
 708         void __iomem            *tx_reg;
 709         void __iomem            *rx_reg;
 710         void __iomem            *chstat_reg;
 711         int                     word_len;
 712
 713         count = xfer->len;
 714         c = count;
 715         word_len = cs->word_len;
 716
 717         l = mcspi_cached_chconf0(spi);
 718
 719         /* We store the pre-calculated register addresses on stack to speed
 720          * up the transfer loop. */
 721         tx_reg          = base + OMAP2_MCSPI_TX0;
 722         rx_reg          = base + OMAP2_MCSPI_RX0;
 723         chstat_reg      = base + OMAP2_MCSPI_CHSTAT0;
 724
 725         if (c < (word_len>>3))
 726                 return 0;
 727
 728         if (word_len <= 8) {
 729                 u8              *rx;
 730                 const u8        *tx;
 731
 732                 rx = xfer->rx_buf;
 733                 tx = xfer->tx_buf;
 734
 735                 do {
 736                         c -= 1;
 737                         if (tx != NULL) {
 738                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 739                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 740                                         dev_err(&spi->dev, "TXS timed out\n");
 741                                         goto out;
 742                                 }
 743                                 dev_vdbg(&spi->dev, "write-%d %02x\n",
 744                                                 word_len, *tx);
 745                                 writel_relaxed(*tx++, tx_reg);
 746                         }
 747                         if (rx != NULL) {
 748                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 749                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 750                                         dev_err(&spi->dev, "RXS timed out\n");
 751                                         goto out;
 752                                 }
 753
 754                                 if (c == 1 && tx == NULL &&
 755                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 756                                         omap2_mcspi_set_enable(spi, 0);
 757                                         *rx++ = readl_relaxed(rx_reg);
 758                                         dev_vdbg(&spi->dev, "read-%d %02x\n",
 759                                                     word_len, *(rx - 1));
 760                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 761                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 762                                                 dev_err(&spi->dev,
 763                                                         "RXS timed out\n");
 764                                                 goto out;
 765                                         }
 766                                         c = 0;
 767                                 } else if (c == 0 && tx == NULL) {
 768                                         omap2_mcspi_set_enable(spi, 0);
 769                                 }
 770
 771                                 *rx++ = readl_relaxed(rx_reg);
 772                                 dev_vdbg(&spi->dev, "read-%d %02x\n",
 773                                                 word_len, *(rx - 1));
 774                         }
 775                         /* Add word delay between each word */
 776                         spi_delay_exec(&xfer->word_delay, xfer);
 777                 } while (c);
 778         } else if (word_len <= 16) {
 779                 u16             *rx;
 780                 const u16       *tx;
 781
 782                 rx = xfer->rx_buf;
 783                 tx = xfer->tx_buf;
 784                 do {
 785                         c -= 2;
 786                         if (tx != NULL) {
 787                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 788                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 789                                         dev_err(&spi->dev, "TXS timed out\n");
 790                                         goto out;
 791                                 }
 792                                 dev_vdbg(&spi->dev, "write-%d %04x\n",
 793                                                 word_len, *tx);
 794                                 writel_relaxed(*tx++, tx_reg);
 795                         }
 796                         if (rx != NULL) {
 797                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 798                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 799                                         dev_err(&spi->dev, "RXS timed out\n");
 800                                         goto out;
 801                                 }
 802
 803                                 if (c == 2 && tx == NULL &&
 804                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 805                                         omap2_mcspi_set_enable(spi, 0);
 806                                         *rx++ = readl_relaxed(rx_reg);
 807                                         dev_vdbg(&spi->dev, "read-%d %04x\n",
 808                                                     word_len, *(rx - 1));
 809                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 810                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 811                                                 dev_err(&spi->dev,
 812                                                         "RXS timed out\n");
 813                                                 goto out;
 814                                         }
 815                                         c = 0;
 816                                 } else if (c == 0 && tx == NULL) {
 817                                         omap2_mcspi_set_enable(spi, 0);
 818                                 }
 819
 820                                 *rx++ = readl_relaxed(rx_reg);
 821                                 dev_vdbg(&spi->dev, "read-%d %04x\n",
 822                                                 word_len, *(rx - 1));
 823                         }
 824                         /* Add word delay between each word */
 825                         spi_delay_exec(&xfer->word_delay, xfer);
 826                 } while (c >= 2);
 827         } else if (word_len <= 32) {
 828                 u32             *rx;
 829                 const u32       *tx;
 830
 831                 rx = xfer->rx_buf;
 832                 tx = xfer->tx_buf;
 833                 do {
 834                         c -= 4;
 835                         if (tx != NULL) {
 836                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 837                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 838                                         dev_err(&spi->dev, "TXS timed out\n");
 839                                         goto out;
 840                                 }
 841                                 dev_vdbg(&spi->dev, "write-%d %08x\n",
 842                                                 word_len, *tx);
 843                                 writel_relaxed(*tx++, tx_reg);
 844                         }
 845                         if (rx != NULL) {
 846                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 847                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 848                                         dev_err(&spi->dev, "RXS timed out\n");
 849                                         goto out;
 850                                 }
 851
 852                                 if (c == 4 && tx == NULL &&
 853                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 854                                         omap2_mcspi_set_enable(spi, 0);
 855                                         *rx++ = readl_relaxed(rx_reg);
 856                                         dev_vdbg(&spi->dev, "read-%d %08x\n",
 857                                                     word_len, *(rx - 1));
 858                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 859                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 860                                                 dev_err(&spi->dev,
 861                                                         "RXS timed out\n");
 862                                                 goto out;
 863                                         }
 864                                         c = 0;
 865                                 } else if (c == 0 && tx == NULL) {
 866                                         omap2_mcspi_set_enable(spi, 0);
 867                                 }
 868
 869                                 *rx++ = readl_relaxed(rx_reg);
 870                                 dev_vdbg(&spi->dev, "read-%d %08x\n",
 871                                                 word_len, *(rx - 1));
 872                         }
 873                         /* Add word delay between each word */
 874                         spi_delay_exec(&xfer->word_delay, xfer);
 875                 } while (c >= 4);
 876         }
 877
 878         /* for TX_ONLY mode, be sure all words have shifted out */
 879         if (xfer->rx_buf == NULL) {
 880                 if (mcspi_wait_for_reg_bit(chstat_reg,
 881                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 882                         dev_err(&spi->dev, "TXS timed out\n");
 883                 } else if (mcspi_wait_for_reg_bit(chstat_reg,
 884                                 OMAP2_MCSPI_CHSTAT_EOT) < 0)
 885                         dev_err(&spi->dev, "EOT timed out\n");
 886
 887                 /* disable chan to purge rx datas received in TX_ONLY transfer,
 888                  * otherwise these rx datas will affect the direct following
 889                  * RX_ONLY transfer.
 890                  */
 891                 omap2_mcspi_set_enable(spi, 0);
 892         }
 893 out:
 894         omap2_mcspi_set_enable(spi, 1);
 895         return count - c;
 896 }
 897
 898 static u32 omap2_mcspi_calc_divisor(u32 speed_hz, u32 ref_clk_hz)
 899 {
 900         u32 div;
 901
 902         for (div = 0; div < 15; div++)
 903                 if (speed_hz >= (ref_clk_hz >> div))
 904                         return div;
 905
 906         return 15;
 907 }
 908
 909 /* called only when no transfer is active to this device */
 910 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 911                 struct spi_transfer *t)
 912 {
 913         struct omap2_mcspi_cs *cs = spi->controller_state;
 914         struct omap2_mcspi *mcspi;
 915         u32 ref_clk_hz, l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 916         u8 word_len = spi->bits_per_word;
 917         u32 speed_hz = spi->max_speed_hz;
 918
 919         mcspi = spi_controller_get_devdata(spi->controller);
 920
 921         if (t != NULL && t->bits_per_word)
 922                 word_len = t->bits_per_word;
 923
 924         cs->word_len = word_len;
 925
 926         if (t && t->speed_hz)
 927                 speed_hz = t->speed_hz;
 928
 929         ref_clk_hz = mcspi->ref_clk_hz;
 930         speed_hz = min_t(u32, speed_hz, ref_clk_hz);
 931         if (speed_hz < (ref_clk_hz / OMAP2_MCSPI_MAX_DIVIDER)) {
 932                 clkd = omap2_mcspi_calc_divisor(speed_hz, ref_clk_hz);
 933                 speed_hz = ref_clk_hz >> clkd;
 934                 clkg = 0;
 935         } else {
 936                 div = (ref_clk_hz + speed_hz - 1) / speed_hz;
 937                 speed_hz = ref_clk_hz / div;
 938                 clkd = (div - 1) & 0xf;
 939                 extclk = (div - 1) >> 4;
 940                 clkg = OMAP2_MCSPI_CHCONF_CLKG;
 941         }
 942
 943         l = mcspi_cached_chconf0(spi);
 944
 945         /* standard 4-wire host mode:  SCK, MOSI/out, MISO/in, nCS
 946          * REVISIT: this controller could support SPI_3WIRE mode.
 947          */
 948         if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 949                 l &= ~OMAP2_MCSPI_CHCONF_IS;
 950                 l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 951                 l |= OMAP2_MCSPI_CHCONF_DPE0;
 952         } else {
 953                 l |= OMAP2_MCSPI_CHCONF_IS;
 954                 l |= OMAP2_MCSPI_CHCONF_DPE1;
 955                 l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 956         }
 957
 958         /* wordlength */
 959         l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 960         l |= (word_len - 1) << 7;
 961
 962         /* set chipselect polarity; manage with FORCE */
 963         if (!(spi->mode & SPI_CS_HIGH))
 964                 l |= OMAP2_MCSPI_CHCONF_EPOL;   /* active-low; normal */
 965         else
 966                 l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 967
 968         /* set clock divisor */
 969         l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 970         l |= clkd << 2;
 971
 972         /* set clock granularity */
 973         l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 974         l |= clkg;
 975         if (clkg) {
 976                 cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 977                 cs->chctrl0 |= extclk << 8;
 978                 mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 979         }
 980
 981         /* set SPI mode 0..3 */
 982         if (spi->mode & SPI_CPOL)
 983                 l |= OMAP2_MCSPI_CHCONF_POL;
 984         else
 985                 l &= ~OMAP2_MCSPI_CHCONF_POL;
 986         if (spi->mode & SPI_CPHA)
 987                 l |= OMAP2_MCSPI_CHCONF_PHA;
 988         else
 989                 l &= ~OMAP2_MCSPI_CHCONF_PHA;
 990
 991         mcspi_write_chconf0(spi, l);
 992
 993         cs->mode = spi->mode;
 994
 995         dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 996                         speed_hz,
 997                         (spi->mode & SPI_CPHA) ? "trailing" : "leading",
 998                         (spi->mode & SPI_CPOL) ? "inverted" : "normal");
 999
1000         return 0;
1001 }
1002
1003 /*
1004  * Note that we currently allow DMA only if we get a channel
1005  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
1006  */
1007 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
1008                                    struct omap2_mcspi_dma *mcspi_dma)
1009 {
1010         int ret = 0;
1011
1012         mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
1013                                              mcspi_dma->dma_rx_ch_name);
1014         if (IS_ERR(mcspi_dma->dma_rx)) {
1015                 ret = PTR_ERR(mcspi_dma->dma_rx);
1016                 mcspi_dma->dma_rx = NULL;
1017                 goto no_dma;
1018         }
1019
1020         mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
1021                                              mcspi_dma->dma_tx_ch_name);
1022         if (IS_ERR(mcspi_dma->dma_tx)) {
1023                 ret = PTR_ERR(mcspi_dma->dma_tx);
1024                 mcspi_dma->dma_tx = NULL;
1025                 dma_release_channel(mcspi_dma->dma_rx);
1026                 mcspi_dma->dma_rx = NULL;
1027         }
1028
1029         init_completion(&mcspi_dma->dma_rx_completion);
1030         init_completion(&mcspi_dma->dma_tx_completion);
1031
1032 no_dma:
1033         return ret;
1034 }
1035
1036 static void omap2_mcspi_release_dma(struct spi_controller *ctlr)
1037 {
1038         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1039         struct omap2_mcspi_dma  *mcspi_dma;
1040         int i;
1041
1042         for (i = 0; i < ctlr->num_chipselect; i++) {
1043                 mcspi_dma = &mcspi->dma_channels[i];
1044
1045                 if (mcspi_dma->dma_rx) {
1046                         dma_release_channel(mcspi_dma->dma_rx);
1047                         mcspi_dma->dma_rx = NULL;
1048                 }
1049                 if (mcspi_dma->dma_tx) {
1050                         dma_release_channel(mcspi_dma->dma_tx);
1051                         mcspi_dma->dma_tx = NULL;
1052                 }
1053         }
1054 }
1055
1056 static void omap2_mcspi_cleanup(struct spi_device *spi)
1057 {
1058         struct omap2_mcspi_cs   *cs;
1059
1060         if (spi->controller_state) {
1061                 /* Unlink controller state from context save list */
1062                 cs = spi->controller_state;
1063                 list_del(&cs->node);
1064
1065                 kfree(cs);
1066         }
1067 }
1068
1069 static int omap2_mcspi_setup(struct spi_device *spi)
1070 {
1071         bool                    initial_setup = false;
1072         int                     ret;
1073         struct omap2_mcspi      *mcspi = spi_controller_get_devdata(spi->controller);
1074         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1075         struct omap2_mcspi_cs   *cs = spi->controller_state;
1076
1077         if (!cs) {
1078                 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
1079                 if (!cs)
1080                         return -ENOMEM;
1081                 cs->base = mcspi->base + spi_get_chipselect(spi, 0) * 0x14;
1082                 cs->phys = mcspi->phys + spi_get_chipselect(spi, 0) * 0x14;
1083                 cs->mode = 0;
1084                 cs->chconf0 = 0;
1085                 cs->chctrl0 = 0;
1086                 spi->controller_state = cs;
1087                 /* Link this to context save list */
1088                 list_add_tail(&cs->node, &ctx->cs);
1089                 initial_setup = true;
1090         }
1091
1092         ret = pm_runtime_resume_and_get(mcspi->dev);
1093         if (ret < 0) {
1094                 if (initial_setup)
1095                         omap2_mcspi_cleanup(spi);
1096
1097                 return ret;
1098         }
1099
1100         ret = omap2_mcspi_setup_transfer(spi, NULL);
1101         if (ret && initial_setup)
1102                 omap2_mcspi_cleanup(spi);
1103
1104         pm_runtime_mark_last_busy(mcspi->dev);
1105         pm_runtime_put_autosuspend(mcspi->dev);
1106
1107         return ret;
1108 }
1109
1110 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1111 {
1112         struct omap2_mcspi *mcspi = data;
1113         u32 irqstat;
1114
1115         irqstat = mcspi_read_reg(mcspi->ctlr, OMAP2_MCSPI_IRQSTATUS);
1116         if (!irqstat)
1117                 return IRQ_NONE;
1118
1119         /* Disable IRQ and wakeup target xfer task */
1120         mcspi_write_reg(mcspi->ctlr, OMAP2_MCSPI_IRQENABLE, 0);
1121         if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1122                 complete(&mcspi->txdone);
1123
1124         return IRQ_HANDLED;
1125 }
1126
1127 static int omap2_mcspi_target_abort(struct spi_controller *ctlr)
1128 {
1129         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1130         struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1131
1132         mcspi->target_aborted = true;
1133         complete(&mcspi_dma->dma_rx_completion);
1134         complete(&mcspi_dma->dma_tx_completion);
1135         complete(&mcspi->txdone);
1136
1137         return 0;
1138 }
1139
1140 static int omap2_mcspi_transfer_one(struct spi_controller *ctlr,
1141                                     struct spi_device *spi,
1142                                     struct spi_transfer *t)
1143 {
1144
1145         /* We only enable one channel at a time -- the one whose message is
1146          * -- although this controller would gladly
1147          * arbitrate among multiple channels.  This corresponds to "single
1148          * channel" host mode.  As a side effect, we need to manage the
1149          * chipselect with the FORCE bit ... CS != channel enable.
1150          */
1151
1152         struct omap2_mcspi              *mcspi;
1153         struct omap2_mcspi_dma          *mcspi_dma;
1154         struct omap2_mcspi_cs           *cs;
1155         struct omap2_mcspi_device_config *cd;
1156         int                             par_override = 0;
1157         int                             status = 0;
1158         u32                             chconf;
1159
1160         mcspi = spi_controller_get_devdata(ctlr);
1161         mcspi_dma = mcspi->dma_channels + spi_get_chipselect(spi, 0);
1162         cs = spi->controller_state;
1163         cd = spi->controller_data;
1164
1165         /*
1166          * The target driver could have changed spi->mode in which case
1167          * it will be different from cs->mode (the current hardware setup).
1168          * If so, set par_override (even though its not a parity issue) so
1169          * omap2_mcspi_setup_transfer will be called to configure the hardware
1170          * with the correct mode on the first iteration of the loop below.
1171          */
1172         if (spi->mode != cs->mode)
1173                 par_override = 1;
1174
1175         omap2_mcspi_set_enable(spi, 0);
1176
1177         if (spi_get_csgpiod(spi, 0))
1178                 omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1179
1180         if (par_override ||
1181             (t->speed_hz != spi->max_speed_hz) ||
1182             (t->bits_per_word != spi->bits_per_word)) {
1183                 par_override = 1;
1184                 status = omap2_mcspi_setup_transfer(spi, t);
1185                 if (status < 0)
1186                         goto out;
1187                 if (t->speed_hz == spi->max_speed_hz &&
1188                     t->bits_per_word == spi->bits_per_word)
1189                         par_override = 0;
1190         }
1191
1192         chconf = mcspi_cached_chconf0(spi);
1193         chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1194         chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1195
1196         if (t->tx_buf == NULL)
1197                 chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1198         else if (t->rx_buf == NULL)
1199                 chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1200
1201         if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1202                 /* Turbo mode is for more than one word */
1203                 if (t->len > ((cs->word_len + 7) >> 3))
1204                         chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1205         }
1206
1207         mcspi_write_chconf0(spi, chconf);
1208
1209         if (t->len) {
1210                 unsigned        count;
1211
1212                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1213                     spi_xfer_is_dma_mapped(ctlr, spi, t))
1214                         omap2_mcspi_set_fifo(spi, t, 1);
1215
1216                 omap2_mcspi_set_enable(spi, 1);
1217
1218                 /* RX_ONLY mode needs dummy data in TX reg */
1219                 if (t->tx_buf == NULL)
1220                         writel_relaxed(0, cs->base
1221                                         + OMAP2_MCSPI_TX0);
1222
1223                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1224                     spi_xfer_is_dma_mapped(ctlr, spi, t))
1225                         count = omap2_mcspi_txrx_dma(spi, t);
1226                 else
1227                         count = omap2_mcspi_txrx_pio(spi, t);
1228
1229                 if (count != t->len) {
1230                         status = -EIO;
1231                         goto out;
1232                 }
1233         }
1234
1235         omap2_mcspi_set_enable(spi, 0);
1236
1237         if (mcspi->fifo_depth > 0)
1238                 omap2_mcspi_set_fifo(spi, t, 0);
1239
1240 out:
1241         /* Restore defaults if they were overriden */
1242         if (par_override) {
1243                 par_override = 0;
1244                 status = omap2_mcspi_setup_transfer(spi, NULL);
1245         }
1246
1247         omap2_mcspi_set_enable(spi, 0);
1248
1249         if (spi_get_csgpiod(spi, 0))
1250                 omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1251
1252         if (mcspi->fifo_depth > 0 && t)
1253                 omap2_mcspi_set_fifo(spi, t, 0);
1254
1255         return status;
1256 }
1257
1258 static int omap2_mcspi_prepare_message(struct spi_controller *ctlr,
1259                                        struct spi_message *msg)
1260 {
1261         struct omap2_mcspi      *mcspi = spi_controller_get_devdata(ctlr);
1262         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1263         struct omap2_mcspi_cs   *cs;
1264         struct spi_transfer     *tr;
1265         u8 bits_per_word;
1266
1267         /*
1268          * The conditions are strict, it is mandatory to check each transfer of the list to see if
1269          * multi-mode is applicable.
1270          */
1271         mcspi->use_multi_mode = true;
1272         list_for_each_entry(tr, &msg->transfers, transfer_list) {
1273                 if (!tr->bits_per_word)
1274                         bits_per_word = msg->spi->bits_per_word;
1275                 else
1276                         bits_per_word = tr->bits_per_word;
1277
1278                 /*
1279                  * Check if this transfer contains only one word;
1280                  */
1281                 if (bits_per_word < 8 && tr->len == 1) {
1282                         /* multi-mode is applicable, only one word (1..7 bits) */
1283                 } else if (bits_per_word >= 8 && tr->len == bits_per_word / 8) {
1284                         /* multi-mode is applicable, only one word (8..32 bits) */
1285                 } else {
1286                         /* multi-mode is not applicable: more than one word in the transfer */
1287                         mcspi->use_multi_mode = false;
1288                 }
1289
1290                 /* Check if transfer asks to change the CS status after the transfer */
1291                 if (!tr->cs_change)
1292                         mcspi->use_multi_mode = false;
1293
1294                 /*
1295                  * If at least one message is not compatible, switch back to single mode
1296                  *
1297                  * The bits_per_word of certain transfer can be different, but it will have no
1298                  * impact on the signal itself.
1299                  */
1300                 if (!mcspi->use_multi_mode)
1301                         break;
1302         }
1303
1304         omap2_mcspi_set_mode(ctlr);
1305
1306         /* In single mode only a single channel can have the FORCE bit enabled
1307          * in its chconf0 register.
1308          * Scan all channels and disable them except the current one.
1309          * A FORCE can remain from a last transfer having cs_change enabled
1310          *
1311          * In multi mode all FORCE bits must be disabled.
1312          */
1313         list_for_each_entry(cs, &ctx->cs, node) {
1314                 if (msg->spi->controller_state == cs && !mcspi->use_multi_mode) {
1315                         continue;
1316                 }
1317
1318                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1319                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1320                         writel_relaxed(cs->chconf0,
1321                                         cs->base + OMAP2_MCSPI_CHCONF0);
1322                         readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1323                 }
1324         }
1325
1326         return 0;
1327 }
1328
1329 static bool omap2_mcspi_can_dma(struct spi_controller *ctlr,
1330                                 struct spi_device *spi,
1331                                 struct spi_transfer *xfer)
1332 {
1333         struct omap2_mcspi *mcspi = spi_controller_get_devdata(spi->controller);
1334         struct omap2_mcspi_dma *mcspi_dma =
1335                 &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
1336
1337         if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1338                 return false;
1339
1340         if (spi_controller_is_target(ctlr))
1341                 return true;
1342
1343         ctlr->dma_rx = mcspi_dma->dma_rx;
1344         ctlr->dma_tx = mcspi_dma->dma_tx;
1345
1346         return (xfer->len >= DMA_MIN_BYTES);
1347 }
1348
1349 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1350 {
1351         struct omap2_mcspi *mcspi = spi_controller_get_devdata(spi->controller);
1352         struct omap2_mcspi_dma *mcspi_dma =
1353                 &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
1354
1355         if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1356                 return mcspi->max_xfer_len;
1357
1358         return SIZE_MAX;
1359 }
1360
1361 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1362 {
1363         struct spi_controller   *ctlr = mcspi->ctlr;
1364         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1365         int                     ret = 0;
1366
1367         ret = pm_runtime_resume_and_get(mcspi->dev);
1368         if (ret < 0)
1369                 return ret;
1370
1371         mcspi_write_reg(ctlr, OMAP2_MCSPI_WAKEUPENABLE,
1372                         OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1373         ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1374
1375         omap2_mcspi_set_mode(ctlr);
1376         pm_runtime_mark_last_busy(mcspi->dev);
1377         pm_runtime_put_autosuspend(mcspi->dev);
1378         return 0;
1379 }
1380
1381 static int omap_mcspi_runtime_suspend(struct device *dev)
1382 {
1383         int error;
1384
1385         error = pinctrl_pm_select_idle_state(dev);
1386         if (error)
1387                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1388
1389         return 0;
1390 }
1391
1392 /*
1393  * When SPI wake up from off-mode, CS is in activate state. If it was in
1394  * inactive state when driver was suspend, then force it to inactive state at
1395  * wake up.
1396  */
1397 static int omap_mcspi_runtime_resume(struct device *dev)
1398 {
1399         struct spi_controller *ctlr = dev_get_drvdata(dev);
1400         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1401         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1402         struct omap2_mcspi_cs *cs;
1403         int error;
1404
1405         error = pinctrl_pm_select_default_state(dev);
1406         if (error)
1407                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1408
1409         /* McSPI: context restore */
1410         mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1411         mcspi_write_reg(ctlr, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1412
1413         list_for_each_entry(cs, &ctx->cs, node) {
1414                 /*
1415                  * We need to toggle CS state for OMAP take this
1416                  * change in account.
1417                  */
1418                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1419                         cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1420                         writel_relaxed(cs->chconf0,
1421                                        cs->base + OMAP2_MCSPI_CHCONF0);
1422                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1423                         writel_relaxed(cs->chconf0,
1424                                        cs->base + OMAP2_MCSPI_CHCONF0);
1425                 } else {
1426                         writel_relaxed(cs->chconf0,
1427                                        cs->base + OMAP2_MCSPI_CHCONF0);
1428                 }
1429         }
1430
1431         return 0;
1432 }
1433
1434 static struct omap2_mcspi_platform_config omap2_pdata = {
1435         .regs_offset = 0,
1436 };
1437
1438 static struct omap2_mcspi_platform_config omap4_pdata = {
1439         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1440 };
1441
1442 static struct omap2_mcspi_platform_config am654_pdata = {
1443         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1444         .max_xfer_len = SZ_4K - 1,
1445 };
1446
1447 static const struct of_device_id omap_mcspi_of_match[] = {
1448         {
1449                 .compatible = "ti,omap2-mcspi",
1450                 .data = &omap2_pdata,
1451         },
1452         {
1453                 .compatible = "ti,omap4-mcspi",
1454                 .data = &omap4_pdata,
1455         },
1456         {
1457                 .compatible = "ti,am654-mcspi",
1458                 .data = &am654_pdata,
1459         },
1460         { },
1461 };
1462 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1463
1464 static int omap2_mcspi_probe(struct platform_device *pdev)
1465 {
1466         struct spi_controller   *ctlr;
1467         const struct omap2_mcspi_platform_config *pdata;
1468         struct omap2_mcspi      *mcspi;
1469         struct resource         *r;
1470         int                     status = 0, i;
1471         u32                     regs_offset = 0;
1472         struct device_node      *node = pdev->dev.of_node;
1473         const struct of_device_id *match;
1474
1475         if (of_property_read_bool(node, "spi-slave"))
1476                 ctlr = spi_alloc_target(&pdev->dev, sizeof(*mcspi));
1477         else
1478                 ctlr = spi_alloc_host(&pdev->dev, sizeof(*mcspi));
1479         if (!ctlr)
1480                 return -ENOMEM;
1481
1482         /* the spi->mode bits understood by this driver: */
1483         ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1484         ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1485         ctlr->setup = omap2_mcspi_setup;
1486         ctlr->auto_runtime_pm = true;
1487         ctlr->prepare_message = omap2_mcspi_prepare_message;
1488         ctlr->can_dma = omap2_mcspi_can_dma;
1489         ctlr->transfer_one = omap2_mcspi_transfer_one;
1490         ctlr->set_cs = omap2_mcspi_set_cs;
1491         ctlr->cleanup = omap2_mcspi_cleanup;
1492         ctlr->target_abort = omap2_mcspi_target_abort;
1493         ctlr->dev.of_node = node;
1494         ctlr->use_gpio_descriptors = true;
1495
1496         platform_set_drvdata(pdev, ctlr);
1497
1498         mcspi = spi_controller_get_devdata(ctlr);
1499         mcspi->ctlr = ctlr;
1500
1501         match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1502         if (match) {
1503                 u32 num_cs = 1; /* default number of chipselect */
1504                 pdata = match->data;
1505
1506                 of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1507                 ctlr->num_chipselect = num_cs;
1508                 if (of_property_read_bool(node, "ti,pindir-d0-out-d1-in"))
1509                         mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1510         } else {
1511                 pdata = dev_get_platdata(&pdev->dev);
1512                 ctlr->num_chipselect = pdata->num_cs;
1513                 mcspi->pin_dir = pdata->pin_dir;
1514         }
1515         regs_offset = pdata->regs_offset;
1516         if (pdata->max_xfer_len) {
1517                 mcspi->max_xfer_len = pdata->max_xfer_len;
1518                 ctlr->max_transfer_size = omap2_mcspi_max_xfer_size;
1519         }
1520
1521         mcspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
1522         if (IS_ERR(mcspi->base)) {
1523                 status = PTR_ERR(mcspi->base);
1524                 goto free_ctlr;
1525         }
1526         mcspi->phys = r->start + regs_offset;
1527         mcspi->base += regs_offset;
1528
1529         mcspi->dev = &pdev->dev;
1530
1531         INIT_LIST_HEAD(&mcspi->ctx.cs);
1532
1533         mcspi->dma_channels = devm_kcalloc(&pdev->dev, ctlr->num_chipselect,
1534                                            sizeof(struct omap2_mcspi_dma),
1535                                            GFP_KERNEL);
1536         if (mcspi->dma_channels == NULL) {
1537                 status = -ENOMEM;
1538                 goto free_ctlr;
1539         }
1540
1541         for (i = 0; i < ctlr->num_chipselect; i++) {
1542                 sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1543                 sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1544
1545                 status = omap2_mcspi_request_dma(mcspi,
1546                                                  &mcspi->dma_channels[i]);
1547                 if (status == -EPROBE_DEFER)
1548                         goto free_ctlr;
1549         }
1550
1551         status = platform_get_irq(pdev, 0);
1552         if (status < 0)
1553                 goto free_ctlr;
1554         init_completion(&mcspi->txdone);
1555         status = devm_request_irq(&pdev->dev, status,
1556                                   omap2_mcspi_irq_handler, 0, pdev->name,
1557                                   mcspi);
1558         if (status) {
1559                 dev_err(&pdev->dev, "Cannot request IRQ");
1560                 goto free_ctlr;
1561         }
1562
1563         mcspi->ref_clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
1564         if (mcspi->ref_clk)
1565                 mcspi->ref_clk_hz = clk_get_rate(mcspi->ref_clk);
1566         else
1567                 mcspi->ref_clk_hz = OMAP2_MCSPI_MAX_FREQ;
1568         ctlr->max_speed_hz = mcspi->ref_clk_hz;
1569         ctlr->min_speed_hz = mcspi->ref_clk_hz >> 15;
1570
1571         pm_runtime_use_autosuspend(&pdev->dev);
1572         pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1573         pm_runtime_enable(&pdev->dev);
1574
1575         status = omap2_mcspi_controller_setup(mcspi);
1576         if (status < 0)
1577                 goto disable_pm;
1578
1579         status = devm_spi_register_controller(&pdev->dev, ctlr);
1580         if (status < 0)
1581                 goto disable_pm;
1582
1583         return status;
1584
1585 disable_pm:
1586         pm_runtime_dont_use_autosuspend(&pdev->dev);
1587         pm_runtime_put_sync(&pdev->dev);
1588         pm_runtime_disable(&pdev->dev);
1589 free_ctlr:
1590         omap2_mcspi_release_dma(ctlr);
1591         spi_controller_put(ctlr);
1592         return status;
1593 }
1594
1595 static void omap2_mcspi_remove(struct platform_device *pdev)
1596 {
1597         struct spi_controller *ctlr = platform_get_drvdata(pdev);
1598         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1599
1600         omap2_mcspi_release_dma(ctlr);
1601
1602         pm_runtime_dont_use_autosuspend(mcspi->dev);
1603         pm_runtime_put_sync(mcspi->dev);
1604         pm_runtime_disable(&pdev->dev);
1605 }
1606
1607 /* work with hotplug and coldplug */
1608 MODULE_ALIAS("platform:omap2_mcspi");
1609
1610 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1611 {
1612         struct spi_controller *ctlr = dev_get_drvdata(dev);
1613         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1614         int error;
1615
1616         error = pinctrl_pm_select_sleep_state(dev);
1617         if (error)
1618                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1619                          __func__, error);
1620
1621         error = spi_controller_suspend(ctlr);
1622         if (error)
1623                 dev_warn(mcspi->dev, "%s: controller suspend failed: %i\n",
1624                          __func__, error);
1625
1626         return pm_runtime_force_suspend(dev);
1627 }
1628
1629 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1630 {
1631         struct spi_controller *ctlr = dev_get_drvdata(dev);
1632         struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1633         int error;
1634
1635         error = spi_controller_resume(ctlr);
1636         if (error)
1637                 dev_warn(mcspi->dev, "%s: controller resume failed: %i\n",
1638                          __func__, error);
1639
1640         return pm_runtime_force_resume(dev);
1641 }
1642
1643 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1644         SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1645                                 omap2_mcspi_resume)
1646         .runtime_suspend        = omap_mcspi_runtime_suspend,
1647         .runtime_resume         = omap_mcspi_runtime_resume,
1648 };
1649
1650 static struct platform_driver omap2_mcspi_driver = {
1651         .driver = {
1652                 .name =         "omap2_mcspi",
1653                 .pm =           &omap2_mcspi_pm_ops,
1654                 .of_match_table = omap_mcspi_of_match,
1655         },
1656         .probe =        omap2_mcspi_probe,
1657         .remove_new =   omap2_mcspi_remove,
1658 };
1659
1660 module_platform_driver(omap2_mcspi_driver);
1661 MODULE_DESCRIPTION("OMAP2 McSPI controller driver");
1662 MODULE_LICENSE("GPL");