drivers/spi/spi-dw-core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Designware SPI core controller driver (refer pxa2xx_spi.c)
   4  *
   5  * Copyright (c) 2009, Intel Corporation.
   6  */
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/bitops.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/preempt.h>
  14 #include <linux/highmem.h>
  15 #include <linux/delay.h>
  16 #include <linux/slab.h>
  17 #include <linux/spi/spi.h>
  18 #include <linux/spi/spi-mem.h>
  19 #include <linux/string.h>
  20 #include <linux/of.h>
  21
  22 #include "internals.h"
  23 #include "spi-dw.h"
  24
  25 #ifdef CONFIG_DEBUG_FS
  26 #include <linux/debugfs.h>
  27 #endif
  28
  29 /* Slave spi_device related */
  30 struct dw_spi_chip_data {
  31         u32 cr0;
  32         u32 rx_sample_dly;      /* RX sample delay */
  33 };
  34
  35 #ifdef CONFIG_DEBUG_FS
  36
  37 #define DW_SPI_DBGFS_REG(_name, _off)   \
  38 {                                       \
  39         .name = _name,                  \
  40         .offset = _off,                 \
  41 }
  42
  43 static const struct debugfs_reg32 dw_spi_dbgfs_regs[] = {
  44         DW_SPI_DBGFS_REG("CTRLR0", DW_SPI_CTRLR0),
  45         DW_SPI_DBGFS_REG("CTRLR1", DW_SPI_CTRLR1),
  46         DW_SPI_DBGFS_REG("SSIENR", DW_SPI_SSIENR),
  47         DW_SPI_DBGFS_REG("SER", DW_SPI_SER),
  48         DW_SPI_DBGFS_REG("BAUDR", DW_SPI_BAUDR),
  49         DW_SPI_DBGFS_REG("TXFTLR", DW_SPI_TXFTLR),
  50         DW_SPI_DBGFS_REG("RXFTLR", DW_SPI_RXFTLR),
  51         DW_SPI_DBGFS_REG("TXFLR", DW_SPI_TXFLR),
  52         DW_SPI_DBGFS_REG("RXFLR", DW_SPI_RXFLR),
  53         DW_SPI_DBGFS_REG("SR", DW_SPI_SR),
  54         DW_SPI_DBGFS_REG("IMR", DW_SPI_IMR),
  55         DW_SPI_DBGFS_REG("ISR", DW_SPI_ISR),
  56         DW_SPI_DBGFS_REG("DMACR", DW_SPI_DMACR),
  57         DW_SPI_DBGFS_REG("DMATDLR", DW_SPI_DMATDLR),
  58         DW_SPI_DBGFS_REG("DMARDLR", DW_SPI_DMARDLR),
  59         DW_SPI_DBGFS_REG("RX_SAMPLE_DLY", DW_SPI_RX_SAMPLE_DLY),
  60 };
  61
  62 static void dw_spi_debugfs_init(struct dw_spi *dws)
  63 {
  64         char name[32];
  65
  66         snprintf(name, 32, "dw_spi%d", dws->host->bus_num);
  67         dws->debugfs = debugfs_create_dir(name, NULL);
  68
  69         dws->regset.regs = dw_spi_dbgfs_regs;
  70         dws->regset.nregs = ARRAY_SIZE(dw_spi_dbgfs_regs);
  71         dws->regset.base = dws->regs;
  72         debugfs_create_regset32("registers", 0400, dws->debugfs, &dws->regset);
  73 }
  74
  75 static void dw_spi_debugfs_remove(struct dw_spi *dws)
  76 {
  77         debugfs_remove_recursive(dws->debugfs);
  78 }
  79
  80 #else
  81 static inline void dw_spi_debugfs_init(struct dw_spi *dws)
  82 {
  83 }
  84
  85 static inline void dw_spi_debugfs_remove(struct dw_spi *dws)
  86 {
  87 }
  88 #endif /* CONFIG_DEBUG_FS */
  89
  90 void dw_spi_set_cs(struct spi_device *spi, bool enable)
  91 {
  92         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
  93         bool cs_high = !!(spi->mode & SPI_CS_HIGH);
  94
  95         /*
  96          * DW SPI controller demands any native CS being set in order to
  97          * proceed with data transfer. So in order to activate the SPI
  98          * communications we must set a corresponding bit in the Slave
  99          * Enable register no matter whether the SPI core is configured to
 100          * support active-high or active-low CS level.
 101          */
 102         if (cs_high == enable)
 103                 dw_writel(dws, DW_SPI_SER, BIT(spi_get_chipselect(spi, 0)));
 104         else
 105                 dw_writel(dws, DW_SPI_SER, 0);
 106 }
 107 EXPORT_SYMBOL_NS_GPL(dw_spi_set_cs, SPI_DW_CORE);
 108
 109 /* Return the max entries we can fill into tx fifo */
 110 static inline u32 dw_spi_tx_max(struct dw_spi *dws)
 111 {
 112         u32 tx_room, rxtx_gap;
 113
 114         tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR);
 115
 116         /*
 117          * Another concern is about the tx/rx mismatch, we
 118          * though to use (dws->fifo_len - rxflr - txflr) as
 119          * one maximum value for tx, but it doesn't cover the
 120          * data which is out of tx/rx fifo and inside the
 121          * shift registers. So a control from sw point of
 122          * view is taken.
 123          */
 124         rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len);
 125
 126         return min3((u32)dws->tx_len, tx_room, rxtx_gap);
 127 }
 128
 129 /* Return the max entries we should read out of rx fifo */
 130 static inline u32 dw_spi_rx_max(struct dw_spi *dws)
 131 {
 132         return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR));
 133 }
 134
 135 static void dw_writer(struct dw_spi *dws)
 136 {
 137         u32 max = dw_spi_tx_max(dws);
 138         u32 txw = 0;
 139
 140         while (max--) {
 141                 if (dws->tx) {
 142                         if (dws->n_bytes == 1)
 143                                 txw = *(u8 *)(dws->tx);
 144                         else if (dws->n_bytes == 2)
 145                                 txw = *(u16 *)(dws->tx);
 146                         else
 147                                 txw = *(u32 *)(dws->tx);
 148
 149                         dws->tx += dws->n_bytes;
 150                 }
 151                 dw_write_io_reg(dws, DW_SPI_DR, txw);
 152                 --dws->tx_len;
 153         }
 154 }
 155
 156 static void dw_reader(struct dw_spi *dws)
 157 {
 158         u32 max = dw_spi_rx_max(dws);
 159         u32 rxw;
 160
 161         while (max--) {
 162                 rxw = dw_read_io_reg(dws, DW_SPI_DR);
 163                 if (dws->rx) {
 164                         if (dws->n_bytes == 1)
 165                                 *(u8 *)(dws->rx) = rxw;
 166                         else if (dws->n_bytes == 2)
 167                                 *(u16 *)(dws->rx) = rxw;
 168                         else
 169                                 *(u32 *)(dws->rx) = rxw;
 170
 171                         dws->rx += dws->n_bytes;
 172                 }
 173                 --dws->rx_len;
 174         }
 175 }
 176
 177 int dw_spi_check_status(struct dw_spi *dws, bool raw)
 178 {
 179         u32 irq_status;
 180         int ret = 0;
 181
 182         if (raw)
 183                 irq_status = dw_readl(dws, DW_SPI_RISR);
 184         else
 185                 irq_status = dw_readl(dws, DW_SPI_ISR);
 186
 187         if (irq_status & DW_SPI_INT_RXOI) {
 188                 dev_err(&dws->host->dev, "RX FIFO overflow detected\n");
 189                 ret = -EIO;
 190         }
 191
 192         if (irq_status & DW_SPI_INT_RXUI) {
 193                 dev_err(&dws->host->dev, "RX FIFO underflow detected\n");
 194                 ret = -EIO;
 195         }
 196
 197         if (irq_status & DW_SPI_INT_TXOI) {
 198                 dev_err(&dws->host->dev, "TX FIFO overflow detected\n");
 199                 ret = -EIO;
 200         }
 201
 202         /* Generically handle the erroneous situation */
 203         if (ret) {
 204                 dw_spi_reset_chip(dws);
 205                 if (dws->host->cur_msg)
 206                         dws->host->cur_msg->status = ret;
 207         }
 208
 209         return ret;
 210 }
 211 EXPORT_SYMBOL_NS_GPL(dw_spi_check_status, SPI_DW_CORE);
 212
 213 static irqreturn_t dw_spi_transfer_handler(struct dw_spi *dws)
 214 {
 215         u16 irq_status = dw_readl(dws, DW_SPI_ISR);
 216
 217         if (dw_spi_check_status(dws, false)) {
 218                 spi_finalize_current_transfer(dws->host);
 219                 return IRQ_HANDLED;
 220         }
 221
 222         /*
 223          * Read data from the Rx FIFO every time we've got a chance executing
 224          * this method. If there is nothing left to receive, terminate the
 225          * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a
 226          * final stage of the transfer. By doing so we'll get the next IRQ
 227          * right when the leftover incoming data is received.
 228          */
 229         dw_reader(dws);
 230         if (!dws->rx_len) {
 231                 dw_spi_mask_intr(dws, 0xff);
 232                 spi_finalize_current_transfer(dws->host);
 233         } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) {
 234                 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1);
 235         }
 236
 237         /*
 238          * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be
 239          * disabled after the data transmission is finished so not to
 240          * have the TXE IRQ flood at the final stage of the transfer.
 241          */
 242         if (irq_status & DW_SPI_INT_TXEI) {
 243                 dw_writer(dws);
 244                 if (!dws->tx_len)
 245                         dw_spi_mask_intr(dws, DW_SPI_INT_TXEI);
 246         }
 247
 248         return IRQ_HANDLED;
 249 }
 250
 251 static irqreturn_t dw_spi_irq(int irq, void *dev_id)
 252 {
 253         struct spi_controller *host = dev_id;
 254         struct dw_spi *dws = spi_controller_get_devdata(host);
 255         u16 irq_status = dw_readl(dws, DW_SPI_ISR) & DW_SPI_INT_MASK;
 256
 257         if (!irq_status)
 258                 return IRQ_NONE;
 259
 260         if (!host->cur_msg) {
 261                 dw_spi_mask_intr(dws, 0xff);
 262                 return IRQ_HANDLED;
 263         }
 264
 265         return dws->transfer_handler(dws);
 266 }
 267
 268 static u32 dw_spi_prepare_cr0(struct dw_spi *dws, struct spi_device *spi)
 269 {
 270         u32 cr0 = 0;
 271
 272         if (dw_spi_ip_is(dws, PSSI)) {
 273                 /* CTRLR0[ 5: 4] Frame Format */
 274                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 275
 276                 /*
 277                  * SPI mode (SCPOL|SCPH)
 278                  * CTRLR0[ 6] Serial Clock Phase
 279                  * CTRLR0[ 7] Serial Clock Polarity
 280                  */
 281                 if (spi->mode & SPI_CPOL)
 282                         cr0 |= DW_PSSI_CTRLR0_SCPOL;
 283                 if (spi->mode & SPI_CPHA)
 284                         cr0 |= DW_PSSI_CTRLR0_SCPHA;
 285
 286                 /* CTRLR0[11] Shift Register Loop */
 287                 if (spi->mode & SPI_LOOP)
 288                         cr0 |= DW_PSSI_CTRLR0_SRL;
 289         } else {
 290                 /* CTRLR0[ 7: 6] Frame Format */
 291                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 292
 293                 /*
 294                  * SPI mode (SCPOL|SCPH)
 295                  * CTRLR0[ 8] Serial Clock Phase
 296                  * CTRLR0[ 9] Serial Clock Polarity
 297                  */
 298                 if (spi->mode & SPI_CPOL)
 299                         cr0 |= DW_HSSI_CTRLR0_SCPOL;
 300                 if (spi->mode & SPI_CPHA)
 301                         cr0 |= DW_HSSI_CTRLR0_SCPHA;
 302
 303                 /* CTRLR0[13] Shift Register Loop */
 304                 if (spi->mode & SPI_LOOP)
 305                         cr0 |= DW_HSSI_CTRLR0_SRL;
 306
 307                 /* CTRLR0[31] MST */
 308                 if (dw_spi_ver_is_ge(dws, HSSI, 102A))
 309                         cr0 |= DW_HSSI_CTRLR0_MST;
 310         }
 311
 312         return cr0;
 313 }
 314
 315 void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi,
 316                           struct dw_spi_cfg *cfg)
 317 {
 318         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 319         u32 cr0 = chip->cr0;
 320         u32 speed_hz;
 321         u16 clk_div;
 322
 323         /* CTRLR0[ 4/3: 0] or CTRLR0[ 20: 16] Data Frame Size */
 324         cr0 |= (cfg->dfs - 1) << dws->dfs_offset;
 325
 326         if (dw_spi_ip_is(dws, PSSI))
 327                 /* CTRLR0[ 9:8] Transfer Mode */
 328                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 329         else
 330                 /* CTRLR0[11:10] Transfer Mode */
 331                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 332
 333         dw_writel(dws, DW_SPI_CTRLR0, cr0);
 334
 335         if (cfg->tmode == DW_SPI_CTRLR0_TMOD_EPROMREAD ||
 336             cfg->tmode == DW_SPI_CTRLR0_TMOD_RO)
 337                 dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0);
 338
 339         /* Note DW APB SSI clock divider doesn't support odd numbers */
 340         clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe;
 341         speed_hz = dws->max_freq / clk_div;
 342
 343         if (dws->current_freq != speed_hz) {
 344                 dw_spi_set_clk(dws, clk_div);
 345                 dws->current_freq = speed_hz;
 346         }
 347
 348         /* Update RX sample delay if required */
 349         if (dws->cur_rx_sample_dly != chip->rx_sample_dly) {
 350                 dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly);
 351                 dws->cur_rx_sample_dly = chip->rx_sample_dly;
 352         }
 353 }
 354 EXPORT_SYMBOL_NS_GPL(dw_spi_update_config, SPI_DW_CORE);
 355
 356 static void dw_spi_irq_setup(struct dw_spi *dws)
 357 {
 358         u16 level;
 359         u8 imask;
 360
 361         /*
 362          * Originally Tx and Rx data lengths match. Rx FIFO Threshold level
 363          * will be adjusted at the final stage of the IRQ-based SPI transfer
 364          * execution so not to lose the leftover of the incoming data.
 365          */
 366         level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len);
 367         dw_writel(dws, DW_SPI_TXFTLR, level);
 368         dw_writel(dws, DW_SPI_RXFTLR, level - 1);
 369
 370         dws->transfer_handler = dw_spi_transfer_handler;
 371
 372         imask = DW_SPI_INT_TXEI | DW_SPI_INT_TXOI |
 373                 DW_SPI_INT_RXUI | DW_SPI_INT_RXOI | DW_SPI_INT_RXFI;
 374         dw_spi_umask_intr(dws, imask);
 375 }
 376
 377 /*
 378  * The iterative procedure of the poll-based transfer is simple: write as much
 379  * as possible to the Tx FIFO, wait until the pending to receive data is ready
 380  * to be read, read it from the Rx FIFO and check whether the performed
 381  * procedure has been successful.
 382  *
 383  * Note this method the same way as the IRQ-based transfer won't work well for
 384  * the SPI devices connected to the controller with native CS due to the
 385  * automatic CS assertion/de-assertion.
 386  */
 387 static int dw_spi_poll_transfer(struct dw_spi *dws,
 388                                 struct spi_transfer *transfer)
 389 {
 390         struct spi_delay delay;
 391         u16 nbits;
 392         int ret;
 393
 394         delay.unit = SPI_DELAY_UNIT_SCK;
 395         nbits = dws->n_bytes * BITS_PER_BYTE;
 396
 397         do {
 398                 dw_writer(dws);
 399
 400                 delay.value = nbits * (dws->rx_len - dws->tx_len);
 401                 spi_delay_exec(&delay, transfer);
 402
 403                 dw_reader(dws);
 404
 405                 ret = dw_spi_check_status(dws, true);
 406                 if (ret)
 407                         return ret;
 408         } while (dws->rx_len);
 409
 410         return 0;
 411 }
 412
 413 static int dw_spi_transfer_one(struct spi_controller *host,
 414                                struct spi_device *spi,
 415                                struct spi_transfer *transfer)
 416 {
 417         struct dw_spi *dws = spi_controller_get_devdata(host);
 418         struct dw_spi_cfg cfg = {
 419                 .tmode = DW_SPI_CTRLR0_TMOD_TR,
 420                 .dfs = transfer->bits_per_word,
 421                 .freq = transfer->speed_hz,
 422         };
 423         int ret;
 424
 425         dws->dma_mapped = 0;
 426         dws->n_bytes = roundup_pow_of_two(BITS_TO_BYTES(transfer->bits_per_word));
 427         dws->tx = (void *)transfer->tx_buf;
 428         dws->tx_len = transfer->len / dws->n_bytes;
 429         dws->rx = transfer->rx_buf;
 430         dws->rx_len = dws->tx_len;
 431
 432         /* Ensure the data above is visible for all CPUs */
 433         smp_mb();
 434
 435         dw_spi_enable_chip(dws, 0);
 436
 437         dw_spi_update_config(dws, spi, &cfg);
 438
 439         transfer->effective_speed_hz = dws->current_freq;
 440
 441         /* Check if current transfer is a DMA transaction */
 442         dws->dma_mapped = spi_xfer_is_dma_mapped(host, spi, transfer);
 443
 444         /* For poll mode just disable all interrupts */
 445         dw_spi_mask_intr(dws, 0xff);
 446
 447         if (dws->dma_mapped) {
 448                 ret = dws->dma_ops->dma_setup(dws, transfer);
 449                 if (ret)
 450                         return ret;
 451         }
 452
 453         dw_spi_enable_chip(dws, 1);
 454
 455         if (dws->dma_mapped)
 456                 return dws->dma_ops->dma_transfer(dws, transfer);
 457         else if (dws->irq == IRQ_NOTCONNECTED)
 458                 return dw_spi_poll_transfer(dws, transfer);
 459
 460         dw_spi_irq_setup(dws);
 461
 462         return 1;
 463 }
 464
 465 static void dw_spi_handle_err(struct spi_controller *host,
 466                               struct spi_message *msg)
 467 {
 468         struct dw_spi *dws = spi_controller_get_devdata(host);
 469
 470         if (dws->dma_mapped)
 471                 dws->dma_ops->dma_stop(dws);
 472
 473         dw_spi_reset_chip(dws);
 474 }
 475
 476 static int dw_spi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 477 {
 478         if (op->data.dir == SPI_MEM_DATA_IN)
 479                 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1);
 480
 481         return 0;
 482 }
 483
 484 static bool dw_spi_supports_mem_op(struct spi_mem *mem,
 485                                    const struct spi_mem_op *op)
 486 {
 487         if (op->data.buswidth > 1 || op->addr.buswidth > 1 ||
 488             op->dummy.buswidth > 1 || op->cmd.buswidth > 1)
 489                 return false;
 490
 491         return spi_mem_default_supports_op(mem, op);
 492 }
 493
 494 static int dw_spi_init_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op)
 495 {
 496         unsigned int i, j, len;
 497         u8 *out;
 498
 499         /*
 500          * Calculate the total length of the EEPROM command transfer and
 501          * either use the pre-allocated buffer or create a temporary one.
 502          */
 503         len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
 504         if (op->data.dir == SPI_MEM_DATA_OUT)
 505                 len += op->data.nbytes;
 506
 507         if (len <= DW_SPI_BUF_SIZE) {
 508                 out = dws->buf;
 509         } else {
 510                 out = kzalloc(len, GFP_KERNEL);
 511                 if (!out)
 512                         return -ENOMEM;
 513         }
 514
 515         /*
 516          * Collect the operation code, address and dummy bytes into the single
 517          * buffer. If it's a transfer with data to be sent, also copy it into the
 518          * single buffer in order to speed the data transmission up.
 519          */
 520         for (i = 0; i < op->cmd.nbytes; ++i)
 521                 out[i] = DW_SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1);
 522         for (j = 0; j < op->addr.nbytes; ++i, ++j)
 523                 out[i] = DW_SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1);
 524         for (j = 0; j < op->dummy.nbytes; ++i, ++j)
 525                 out[i] = 0x0;
 526
 527         if (op->data.dir == SPI_MEM_DATA_OUT)
 528                 memcpy(&out[i], op->data.buf.out, op->data.nbytes);
 529
 530         dws->n_bytes = 1;
 531         dws->tx = out;
 532         dws->tx_len = len;
 533         if (op->data.dir == SPI_MEM_DATA_IN) {
 534                 dws->rx = op->data.buf.in;
 535                 dws->rx_len = op->data.nbytes;
 536         } else {
 537                 dws->rx = NULL;
 538                 dws->rx_len = 0;
 539         }
 540
 541         return 0;
 542 }
 543
 544 static void dw_spi_free_mem_buf(struct dw_spi *dws)
 545 {
 546         if (dws->tx != dws->buf)
 547                 kfree(dws->tx);
 548 }
 549
 550 static int dw_spi_write_then_read(struct dw_spi *dws, struct spi_device *spi)
 551 {
 552         u32 room, entries, sts;
 553         unsigned int len;
 554         u8 *buf;
 555
 556         /*
 557          * At initial stage we just pre-fill the Tx FIFO in with no rush,
 558          * since native CS hasn't been enabled yet and the automatic data
 559          * transmission won't start til we do that.
 560          */
 561         len = min(dws->fifo_len, dws->tx_len);
 562         buf = dws->tx;
 563         while (len--)
 564                 dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 565
 566         /*
 567          * After setting any bit in the SER register the transmission will
 568          * start automatically. We have to keep up with that procedure
 569          * otherwise the CS de-assertion will happen whereupon the memory
 570          * operation will be pre-terminated.
 571          */
 572         len = dws->tx_len - ((void *)buf - dws->tx);
 573         dw_spi_set_cs(spi, false);
 574         while (len) {
 575                 entries = readl_relaxed(dws->regs + DW_SPI_TXFLR);
 576                 if (!entries) {
 577                         dev_err(&dws->host->dev, "CS de-assertion on Tx\n");
 578                         return -EIO;
 579                 }
 580                 room = min(dws->fifo_len - entries, len);
 581                 for (; room; --room, --len)
 582                         dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 583         }
 584
 585         /*
 586          * Data fetching will start automatically if the EEPROM-read mode is
 587          * activated. We have to keep up with the incoming data pace to
 588          * prevent the Rx FIFO overflow causing the inbound data loss.
 589          */
 590         len = dws->rx_len;
 591         buf = dws->rx;
 592         while (len) {
 593                 entries = readl_relaxed(dws->regs + DW_SPI_RXFLR);
 594                 if (!entries) {
 595                         sts = readl_relaxed(dws->regs + DW_SPI_RISR);
 596                         if (sts & DW_SPI_INT_RXOI) {
 597                                 dev_err(&dws->host->dev, "FIFO overflow on Rx\n");
 598                                 return -EIO;
 599                         }
 600                         continue;
 601                 }
 602                 entries = min(entries, len);
 603                 for (; entries; --entries, --len)
 604                         *buf++ = dw_read_io_reg(dws, DW_SPI_DR);
 605         }
 606
 607         return 0;
 608 }
 609
 610 static inline bool dw_spi_ctlr_busy(struct dw_spi *dws)
 611 {
 612         return dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_BUSY;
 613 }
 614
 615 static int dw_spi_wait_mem_op_done(struct dw_spi *dws)
 616 {
 617         int retry = DW_SPI_WAIT_RETRIES;
 618         struct spi_delay delay;
 619         unsigned long ns, us;
 620         u32 nents;
 621
 622         nents = dw_readl(dws, DW_SPI_TXFLR);
 623         ns = NSEC_PER_SEC / dws->current_freq * nents;
 624         ns *= dws->n_bytes * BITS_PER_BYTE;
 625         if (ns <= NSEC_PER_USEC) {
 626                 delay.unit = SPI_DELAY_UNIT_NSECS;
 627                 delay.value = ns;
 628         } else {
 629                 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
 630                 delay.unit = SPI_DELAY_UNIT_USECS;
 631                 delay.value = clamp_val(us, 0, USHRT_MAX);
 632         }
 633
 634         while (dw_spi_ctlr_busy(dws) && retry--)
 635                 spi_delay_exec(&delay, NULL);
 636
 637         if (retry < 0) {
 638                 dev_err(&dws->host->dev, "Mem op hanged up\n");
 639                 return -EIO;
 640         }
 641
 642         return 0;
 643 }
 644
 645 static void dw_spi_stop_mem_op(struct dw_spi *dws, struct spi_device *spi)
 646 {
 647         dw_spi_enable_chip(dws, 0);
 648         dw_spi_set_cs(spi, true);
 649         dw_spi_enable_chip(dws, 1);
 650 }
 651
 652 /*
 653  * The SPI memory operation implementation below is the best choice for the
 654  * devices, which are selected by the native chip-select lane. It's
 655  * specifically developed to workaround the problem with automatic chip-select
 656  * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current
 657  * SPI-mem core calls exec_op() callback only if the GPIO-based CS is
 658  * unavailable.
 659  */
 660 static int dw_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 661 {
 662         struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller);
 663         struct dw_spi_cfg cfg;
 664         unsigned long flags;
 665         int ret;
 666
 667         /*
 668          * Collect the outbound data into a single buffer to speed the
 669          * transmission up at least on the initial stage.
 670          */
 671         ret = dw_spi_init_mem_buf(dws, op);
 672         if (ret)
 673                 return ret;
 674
 675         /*
 676          * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN
 677          * operation. Transmit-only mode is suitable for the rest of them.
 678          */
 679         cfg.dfs = 8;
 680         cfg.freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_mem_freq);
 681         if (op->data.dir == SPI_MEM_DATA_IN) {
 682                 cfg.tmode = DW_SPI_CTRLR0_TMOD_EPROMREAD;
 683                 cfg.ndf = op->data.nbytes;
 684         } else {
 685                 cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
 686         }
 687
 688         dw_spi_enable_chip(dws, 0);
 689
 690         dw_spi_update_config(dws, mem->spi, &cfg);
 691
 692         dw_spi_mask_intr(dws, 0xff);
 693
 694         dw_spi_enable_chip(dws, 1);
 695
 696         /*
 697          * DW APB SSI controller has very nasty peculiarities. First originally
 698          * (without any vendor-specific modifications) it doesn't provide a
 699          * direct way to set and clear the native chip-select signal. Instead
 700          * the controller asserts the CS lane if Tx FIFO isn't empty and a
 701          * transmission is going on, and automatically de-asserts it back to
 702          * the high level if the Tx FIFO doesn't have anything to be pushed
 703          * out. Due to that a multi-tasking or heavy IRQs activity might be
 704          * fatal, since the transfer procedure preemption may cause the Tx FIFO
 705          * getting empty and sudden CS de-assertion, which in the middle of the
 706          * transfer will most likely cause the data loss. Secondly the
 707          * EEPROM-read or Read-only DW SPI transfer modes imply the incoming
 708          * data being automatically pulled in into the Rx FIFO. So if the
 709          * driver software is late in fetching the data from the FIFO before
 710          * it's overflown, new incoming data will be lost. In order to make
 711          * sure the executed memory operations are CS-atomic and to prevent the
 712          * Rx FIFO overflow we have to disable the local interrupts so to block
 713          * any preemption during the subsequent IO operations.
 714          *
 715          * Note. At some circumstances disabling IRQs may not help to prevent
 716          * the problems described above. The CS de-assertion and Rx FIFO
 717          * overflow may still happen due to the relatively slow system bus or
 718          * CPU not working fast enough, so the write-then-read algo implemented
 719          * here just won't keep up with the SPI bus data transfer. Such
 720          * situation is highly platform specific and is supposed to be fixed by
 721          * manually restricting the SPI bus frequency using the
 722          * dws->max_mem_freq parameter.
 723          */
 724         local_irq_save(flags);
 725         preempt_disable();
 726
 727         ret = dw_spi_write_then_read(dws, mem->spi);
 728
 729         local_irq_restore(flags);
 730         preempt_enable();
 731
 732         /*
 733          * Wait for the operation being finished and check the controller
 734          * status only if there hasn't been any run-time error detected. In the
 735          * former case it's just pointless. In the later one to prevent an
 736          * additional error message printing since any hw error flag being set
 737          * would be due to an error detected on the data transfer.
 738          */
 739         if (!ret) {
 740                 ret = dw_spi_wait_mem_op_done(dws);
 741                 if (!ret)
 742                         ret = dw_spi_check_status(dws, true);
 743         }
 744
 745         dw_spi_stop_mem_op(dws, mem->spi);
 746
 747         dw_spi_free_mem_buf(dws);
 748
 749         return ret;
 750 }
 751
 752 /*
 753  * Initialize the default memory operations if a glue layer hasn't specified
 754  * custom ones. Direct mapping operations will be preserved anyway since DW SPI
 755  * controller doesn't have an embedded dirmap interface. Note the memory
 756  * operations implemented in this driver is the best choice only for the DW APB
 757  * SSI controller with standard native CS functionality. If a hardware vendor
 758  * has fixed the automatic CS assertion/de-assertion peculiarity, then it will
 759  * be safer to use the normal SPI-messages-based transfers implementation.
 760  */
 761 static void dw_spi_init_mem_ops(struct dw_spi *dws)
 762 {
 763         if (!dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) &&
 764             !dws->set_cs) {
 765                 dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size;
 766                 dws->mem_ops.supports_op = dw_spi_supports_mem_op;
 767                 dws->mem_ops.exec_op = dw_spi_exec_mem_op;
 768                 if (!dws->max_mem_freq)
 769                         dws->max_mem_freq = dws->max_freq;
 770         }
 771 }
 772
 773 /* This may be called twice for each spi dev */
 774 static int dw_spi_setup(struct spi_device *spi)
 775 {
 776         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 777         struct dw_spi_chip_data *chip;
 778
 779         /* Only alloc on first setup */
 780         chip = spi_get_ctldata(spi);
 781         if (!chip) {
 782                 struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 783                 u32 rx_sample_dly_ns;
 784
 785                 chip = kzalloc(sizeof(*chip), GFP_KERNEL);
 786                 if (!chip)
 787                         return -ENOMEM;
 788                 spi_set_ctldata(spi, chip);
 789                 /* Get specific / default rx-sample-delay */
 790                 if (device_property_read_u32(&spi->dev,
 791                                              "rx-sample-delay-ns",
 792                                              &rx_sample_dly_ns) != 0)
 793                         /* Use default controller value */
 794                         rx_sample_dly_ns = dws->def_rx_sample_dly_ns;
 795                 chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,
 796                                                         NSEC_PER_SEC /
 797                                                         dws->max_freq);
 798         }
 799
 800         /*
 801          * Update CR0 data each time the setup callback is invoked since
 802          * the device parameters could have been changed, for instance, by
 803          * the MMC SPI driver or something else.
 804          */
 805         chip->cr0 = dw_spi_prepare_cr0(dws, spi);
 806
 807         return 0;
 808 }
 809
 810 static void dw_spi_cleanup(struct spi_device *spi)
 811 {
 812         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 813
 814         kfree(chip);
 815         spi_set_ctldata(spi, NULL);
 816 }
 817
 818 /* Restart the controller, disable all interrupts, clean rx fifo */
 819 static void dw_spi_hw_init(struct device *dev, struct dw_spi *dws)
 820 {
 821         dw_spi_reset_chip(dws);
 822
 823         /*
 824          * Retrieve the Synopsys component version if it hasn't been specified
 825          * by the platform. CoreKit version ID is encoded as a 3-chars ASCII
 826          * code enclosed with '*' (typical for the most of Synopsys IP-cores).
 827          */
 828         if (!dws->ver) {
 829                 dws->ver = dw_readl(dws, DW_SPI_VERSION);
 830
 831                 dev_dbg(dev, "Synopsys DWC%sSSI v%c.%c%c\n",
 832                         dw_spi_ip_is(dws, PSSI) ? " APB " : " ",
 833                         DW_SPI_GET_BYTE(dws->ver, 3), DW_SPI_GET_BYTE(dws->ver, 2),
 834                         DW_SPI_GET_BYTE(dws->ver, 1));
 835         }
 836
 837         /*
 838          * Try to detect the number of native chip-selects if the platform
 839          * driver didn't set it up. There can be up to 16 lines configured.
 840          */
 841         if (!dws->num_cs) {
 842                 u32 ser;
 843
 844                 dw_writel(dws, DW_SPI_SER, 0xffff);
 845                 ser = dw_readl(dws, DW_SPI_SER);
 846                 dw_writel(dws, DW_SPI_SER, 0);
 847
 848                 dws->num_cs = hweight16(ser);
 849         }
 850
 851         /*
 852          * Try to detect the FIFO depth if not set by interface driver,
 853          * the depth could be from 2 to 256 from HW spec
 854          */
 855         if (!dws->fifo_len) {
 856                 u32 fifo;
 857
 858                 for (fifo = 1; fifo < 256; fifo++) {
 859                         dw_writel(dws, DW_SPI_TXFTLR, fifo);
 860                         if (fifo != dw_readl(dws, DW_SPI_TXFTLR))
 861                                 break;
 862                 }
 863                 dw_writel(dws, DW_SPI_TXFTLR, 0);
 864
 865                 dws->fifo_len = (fifo == 1) ? 0 : fifo;
 866                 dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len);
 867         }
 868
 869         /*
 870          * Detect CTRLR0.DFS field size and offset by testing the lowest bits
 871          * writability. Note DWC SSI controller also has the extended DFS, but
 872          * with zero offset.
 873          */
 874         if (dw_spi_ip_is(dws, PSSI)) {
 875                 u32 cr0, tmp = dw_readl(dws, DW_SPI_CTRLR0);
 876
 877                 dw_spi_enable_chip(dws, 0);
 878                 dw_writel(dws, DW_SPI_CTRLR0, 0xffffffff);
 879                 cr0 = dw_readl(dws, DW_SPI_CTRLR0);
 880                 dw_writel(dws, DW_SPI_CTRLR0, tmp);
 881                 dw_spi_enable_chip(dws, 1);
 882
 883                 if (!(cr0 & DW_PSSI_CTRLR0_DFS_MASK)) {
 884                         dws->caps |= DW_SPI_CAP_DFS32;
 885                         dws->dfs_offset = __bf_shf(DW_PSSI_CTRLR0_DFS32_MASK);
 886                         dev_dbg(dev, "Detected 32-bits max data frame size\n");
 887                 }
 888         } else {
 889                 dws->caps |= DW_SPI_CAP_DFS32;
 890         }
 891
 892         /* enable HW fixup for explicit CS deselect for Amazon's alpine chip */
 893         if (dws->caps & DW_SPI_CAP_CS_OVERRIDE)
 894                 dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF);
 895 }
 896
 897 int dw_spi_add_host(struct device *dev, struct dw_spi *dws)
 898 {
 899         struct spi_controller *host;
 900         int ret;
 901
 902         if (!dws)
 903                 return -EINVAL;
 904
 905         host = spi_alloc_host(dev, 0);
 906         if (!host)
 907                 return -ENOMEM;
 908
 909         device_set_node(&host->dev, dev_fwnode(dev));
 910
 911         dws->host = host;
 912         dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR);
 913
 914         spi_controller_set_devdata(host, dws);
 915
 916         /* Basic HW init */
 917         dw_spi_hw_init(dev, dws);
 918
 919         ret = request_irq(dws->irq, dw_spi_irq, IRQF_SHARED, dev_name(dev),
 920                           host);
 921         if (ret < 0 && ret != -ENOTCONN) {
 922                 dev_err(dev, "can not get IRQ\n");
 923                 goto err_free_host;
 924         }
 925
 926         dw_spi_init_mem_ops(dws);
 927
 928         host->use_gpio_descriptors = true;
 929         host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;
 930         if (dws->caps & DW_SPI_CAP_DFS32)
 931                 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
 932         else
 933                 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
 934         host->bus_num = dws->bus_num;
 935         host->num_chipselect = dws->num_cs;
 936         host->setup = dw_spi_setup;
 937         host->cleanup = dw_spi_cleanup;
 938         if (dws->set_cs)
 939                 host->set_cs = dws->set_cs;
 940         else
 941                 host->set_cs = dw_spi_set_cs;
 942         host->transfer_one = dw_spi_transfer_one;
 943         host->handle_err = dw_spi_handle_err;
 944         if (dws->mem_ops.exec_op)
 945                 host->mem_ops = &dws->mem_ops;
 946         host->max_speed_hz = dws->max_freq;
 947         host->flags = SPI_CONTROLLER_GPIO_SS;
 948         host->auto_runtime_pm = true;
 949
 950         /* Get default rx sample delay */
 951         device_property_read_u32(dev, "rx-sample-delay-ns",
 952                                  &dws->def_rx_sample_dly_ns);
 953
 954         if (dws->dma_ops && dws->dma_ops->dma_init) {
 955                 ret = dws->dma_ops->dma_init(dev, dws);
 956                 if (ret == -EPROBE_DEFER) {
 957                         goto err_free_irq;
 958                 } else if (ret) {
 959                         dev_warn(dev, "DMA init failed\n");
 960                 } else {
 961                         host->can_dma = dws->dma_ops->can_dma;
 962                         host->flags |= SPI_CONTROLLER_MUST_TX;
 963                 }
 964         }
 965
 966         ret = spi_register_controller(host);
 967         if (ret) {
 968                 dev_err_probe(dev, ret, "problem registering spi host\n");
 969                 goto err_dma_exit;
 970         }
 971
 972         dw_spi_debugfs_init(dws);
 973         return 0;
 974
 975 err_dma_exit:
 976         if (dws->dma_ops && dws->dma_ops->dma_exit)
 977                 dws->dma_ops->dma_exit(dws);
 978         dw_spi_enable_chip(dws, 0);
 979 err_free_irq:
 980         free_irq(dws->irq, host);
 981 err_free_host:
 982         spi_controller_put(host);
 983         return ret;
 984 }
 985 EXPORT_SYMBOL_NS_GPL(dw_spi_add_host, SPI_DW_CORE);
 986
 987 void dw_spi_remove_host(struct dw_spi *dws)
 988 {
 989         dw_spi_debugfs_remove(dws);
 990
 991         spi_unregister_controller(dws->host);
 992
 993         if (dws->dma_ops && dws->dma_ops->dma_exit)
 994                 dws->dma_ops->dma_exit(dws);
 995
 996         dw_spi_shutdown_chip(dws);
 997
 998         free_irq(dws->irq, dws->host);
 999 }
1000 EXPORT_SYMBOL_NS_GPL(dw_spi_remove_host, SPI_DW_CORE);
1001
1002 int dw_spi_suspend_host(struct dw_spi *dws)
1003 {
1004         int ret;
1005
1006         ret = spi_controller_suspend(dws->host);
1007         if (ret)
1008                 return ret;
1009
1010         dw_spi_shutdown_chip(dws);
1011         return 0;
1012 }
1013 EXPORT_SYMBOL_NS_GPL(dw_spi_suspend_host, SPI_DW_CORE);
1014
1015 int dw_spi_resume_host(struct dw_spi *dws)
1016 {
1017         dw_spi_hw_init(&dws->host->dev, dws);
1018         return spi_controller_resume(dws->host);
1019 }
1020 EXPORT_SYMBOL_NS_GPL(dw_spi_resume_host, SPI_DW_CORE);
1021
1022 MODULE_AUTHOR("Feng Tang <[email protected]>");
1023 MODULE_DESCRIPTION("Driver for DesignWare SPI controller core");
1024 MODULE_LICENSE("GPL v2");