drivers/spi/spi-dw-core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Designware SPI core controller driver (refer pxa2xx_spi.c)
   4  *
   5  * Copyright (c) 2009, Intel Corporation.
   6  */
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/bitops.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/preempt.h>
  14 #include <linux/highmem.h>
  15 #include <linux/delay.h>
  16 #include <linux/slab.h>
  17 #include <linux/spi/spi.h>
  18 #include <linux/spi/spi-mem.h>
  19 #include <linux/string.h>
  20 #include <linux/of.h>
  21
  22 #include "spi-dw.h"
  23
  24 #ifdef CONFIG_DEBUG_FS
  25 #include <linux/debugfs.h>
  26 #endif
  27
  28 /* Slave spi_device related */
  29 struct dw_spi_chip_data {
  30         u32 cr0;
  31         u32 rx_sample_dly;      /* RX sample delay */
  32 };
  33
  34 #ifdef CONFIG_DEBUG_FS
  35
  36 #define DW_SPI_DBGFS_REG(_name, _off)   \
  37 {                                       \
  38         .name = _name,                  \
  39         .offset = _off,                 \
  40 }
  41
  42 static const struct debugfs_reg32 dw_spi_dbgfs_regs[] = {
  43         DW_SPI_DBGFS_REG("CTRLR0", DW_SPI_CTRLR0),
  44         DW_SPI_DBGFS_REG("CTRLR1", DW_SPI_CTRLR1),
  45         DW_SPI_DBGFS_REG("SSIENR", DW_SPI_SSIENR),
  46         DW_SPI_DBGFS_REG("SER", DW_SPI_SER),
  47         DW_SPI_DBGFS_REG("BAUDR", DW_SPI_BAUDR),
  48         DW_SPI_DBGFS_REG("TXFTLR", DW_SPI_TXFTLR),
  49         DW_SPI_DBGFS_REG("RXFTLR", DW_SPI_RXFTLR),
  50         DW_SPI_DBGFS_REG("TXFLR", DW_SPI_TXFLR),
  51         DW_SPI_DBGFS_REG("RXFLR", DW_SPI_RXFLR),
  52         DW_SPI_DBGFS_REG("SR", DW_SPI_SR),
  53         DW_SPI_DBGFS_REG("IMR", DW_SPI_IMR),
  54         DW_SPI_DBGFS_REG("ISR", DW_SPI_ISR),
  55         DW_SPI_DBGFS_REG("DMACR", DW_SPI_DMACR),
  56         DW_SPI_DBGFS_REG("DMATDLR", DW_SPI_DMATDLR),
  57         DW_SPI_DBGFS_REG("DMARDLR", DW_SPI_DMARDLR),
  58         DW_SPI_DBGFS_REG("RX_SAMPLE_DLY", DW_SPI_RX_SAMPLE_DLY),
  59 };
  60
  61 static void dw_spi_debugfs_init(struct dw_spi *dws)
  62 {
  63         char name[32];
  64
  65         snprintf(name, 32, "dw_spi%d", dws->host->bus_num);
  66         dws->debugfs = debugfs_create_dir(name, NULL);
  67
  68         dws->regset.regs = dw_spi_dbgfs_regs;
  69         dws->regset.nregs = ARRAY_SIZE(dw_spi_dbgfs_regs);
  70         dws->regset.base = dws->regs;
  71         debugfs_create_regset32("registers", 0400, dws->debugfs, &dws->regset);
  72 }
  73
  74 static void dw_spi_debugfs_remove(struct dw_spi *dws)
  75 {
  76         debugfs_remove_recursive(dws->debugfs);
  77 }
  78
  79 #else
  80 static inline void dw_spi_debugfs_init(struct dw_spi *dws)
  81 {
  82 }
  83
  84 static inline void dw_spi_debugfs_remove(struct dw_spi *dws)
  85 {
  86 }
  87 #endif /* CONFIG_DEBUG_FS */
  88
  89 void dw_spi_set_cs(struct spi_device *spi, bool enable)
  90 {
  91         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
  92         bool cs_high = !!(spi->mode & SPI_CS_HIGH);
  93
  94         /*
  95          * DW SPI controller demands any native CS being set in order to
  96          * proceed with data transfer. So in order to activate the SPI
  97          * communications we must set a corresponding bit in the Slave
  98          * Enable register no matter whether the SPI core is configured to
  99          * support active-high or active-low CS level.
 100          */
 101         if (cs_high == enable)
 102                 dw_writel(dws, DW_SPI_SER, BIT(spi_get_chipselect(spi, 0)));
 103         else
 104                 dw_writel(dws, DW_SPI_SER, 0);
 105 }
 106 EXPORT_SYMBOL_NS_GPL(dw_spi_set_cs, SPI_DW_CORE);
 107
 108 /* Return the max entries we can fill into tx fifo */
 109 static inline u32 dw_spi_tx_max(struct dw_spi *dws)
 110 {
 111         u32 tx_room, rxtx_gap;
 112
 113         tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR);
 114
 115         /*
 116          * Another concern is about the tx/rx mismatch, we
 117          * though to use (dws->fifo_len - rxflr - txflr) as
 118          * one maximum value for tx, but it doesn't cover the
 119          * data which is out of tx/rx fifo and inside the
 120          * shift registers. So a control from sw point of
 121          * view is taken.
 122          */
 123         rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len);
 124
 125         return min3((u32)dws->tx_len, tx_room, rxtx_gap);
 126 }
 127
 128 /* Return the max entries we should read out of rx fifo */
 129 static inline u32 dw_spi_rx_max(struct dw_spi *dws)
 130 {
 131         return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR));
 132 }
 133
 134 static void dw_writer(struct dw_spi *dws)
 135 {
 136         u32 max = dw_spi_tx_max(dws);
 137         u32 txw = 0;
 138
 139         while (max--) {
 140                 if (dws->tx) {
 141                         if (dws->n_bytes == 1)
 142                                 txw = *(u8 *)(dws->tx);
 143                         else if (dws->n_bytes == 2)
 144                                 txw = *(u16 *)(dws->tx);
 145                         else
 146                                 txw = *(u32 *)(dws->tx);
 147
 148                         dws->tx += dws->n_bytes;
 149                 }
 150                 dw_write_io_reg(dws, DW_SPI_DR, txw);
 151                 --dws->tx_len;
 152         }
 153 }
 154
 155 static void dw_reader(struct dw_spi *dws)
 156 {
 157         u32 max = dw_spi_rx_max(dws);
 158         u32 rxw;
 159
 160         while (max--) {
 161                 rxw = dw_read_io_reg(dws, DW_SPI_DR);
 162                 if (dws->rx) {
 163                         if (dws->n_bytes == 1)
 164                                 *(u8 *)(dws->rx) = rxw;
 165                         else if (dws->n_bytes == 2)
 166                                 *(u16 *)(dws->rx) = rxw;
 167                         else
 168                                 *(u32 *)(dws->rx) = rxw;
 169
 170                         dws->rx += dws->n_bytes;
 171                 }
 172                 --dws->rx_len;
 173         }
 174 }
 175
 176 int dw_spi_check_status(struct dw_spi *dws, bool raw)
 177 {
 178         u32 irq_status;
 179         int ret = 0;
 180
 181         if (raw)
 182                 irq_status = dw_readl(dws, DW_SPI_RISR);
 183         else
 184                 irq_status = dw_readl(dws, DW_SPI_ISR);
 185
 186         if (irq_status & DW_SPI_INT_RXOI) {
 187                 dev_err(&dws->host->dev, "RX FIFO overflow detected\n");
 188                 ret = -EIO;
 189         }
 190
 191         if (irq_status & DW_SPI_INT_RXUI) {
 192                 dev_err(&dws->host->dev, "RX FIFO underflow detected\n");
 193                 ret = -EIO;
 194         }
 195
 196         if (irq_status & DW_SPI_INT_TXOI) {
 197                 dev_err(&dws->host->dev, "TX FIFO overflow detected\n");
 198                 ret = -EIO;
 199         }
 200
 201         /* Generically handle the erroneous situation */
 202         if (ret) {
 203                 dw_spi_reset_chip(dws);
 204                 if (dws->host->cur_msg)
 205                         dws->host->cur_msg->status = ret;
 206         }
 207
 208         return ret;
 209 }
 210 EXPORT_SYMBOL_NS_GPL(dw_spi_check_status, SPI_DW_CORE);
 211
 212 static irqreturn_t dw_spi_transfer_handler(struct dw_spi *dws)
 213 {
 214         u16 irq_status = dw_readl(dws, DW_SPI_ISR);
 215
 216         if (dw_spi_check_status(dws, false)) {
 217                 spi_finalize_current_transfer(dws->host);
 218                 return IRQ_HANDLED;
 219         }
 220
 221         /*
 222          * Read data from the Rx FIFO every time we've got a chance executing
 223          * this method. If there is nothing left to receive, terminate the
 224          * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a
 225          * final stage of the transfer. By doing so we'll get the next IRQ
 226          * right when the leftover incoming data is received.
 227          */
 228         dw_reader(dws);
 229         if (!dws->rx_len) {
 230                 dw_spi_mask_intr(dws, 0xff);
 231                 spi_finalize_current_transfer(dws->host);
 232         } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) {
 233                 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1);
 234         }
 235
 236         /*
 237          * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be
 238          * disabled after the data transmission is finished so not to
 239          * have the TXE IRQ flood at the final stage of the transfer.
 240          */
 241         if (irq_status & DW_SPI_INT_TXEI) {
 242                 dw_writer(dws);
 243                 if (!dws->tx_len)
 244                         dw_spi_mask_intr(dws, DW_SPI_INT_TXEI);
 245         }
 246
 247         return IRQ_HANDLED;
 248 }
 249
 250 static irqreturn_t dw_spi_irq(int irq, void *dev_id)
 251 {
 252         struct spi_controller *host = dev_id;
 253         struct dw_spi *dws = spi_controller_get_devdata(host);
 254         u16 irq_status = dw_readl(dws, DW_SPI_ISR) & DW_SPI_INT_MASK;
 255
 256         if (!irq_status)
 257                 return IRQ_NONE;
 258
 259         if (!host->cur_msg) {
 260                 dw_spi_mask_intr(dws, 0xff);
 261                 return IRQ_HANDLED;
 262         }
 263
 264         return dws->transfer_handler(dws);
 265 }
 266
 267 static u32 dw_spi_prepare_cr0(struct dw_spi *dws, struct spi_device *spi)
 268 {
 269         u32 cr0 = 0;
 270
 271         if (dw_spi_ip_is(dws, PSSI)) {
 272                 /* CTRLR0[ 5: 4] Frame Format */
 273                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 274
 275                 /*
 276                  * SPI mode (SCPOL|SCPH)
 277                  * CTRLR0[ 6] Serial Clock Phase
 278                  * CTRLR0[ 7] Serial Clock Polarity
 279                  */
 280                 if (spi->mode & SPI_CPOL)
 281                         cr0 |= DW_PSSI_CTRLR0_SCPOL;
 282                 if (spi->mode & SPI_CPHA)
 283                         cr0 |= DW_PSSI_CTRLR0_SCPHA;
 284
 285                 /* CTRLR0[11] Shift Register Loop */
 286                 if (spi->mode & SPI_LOOP)
 287                         cr0 |= DW_PSSI_CTRLR0_SRL;
 288         } else {
 289                 /* CTRLR0[ 7: 6] Frame Format */
 290                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 291
 292                 /*
 293                  * SPI mode (SCPOL|SCPH)
 294                  * CTRLR0[ 8] Serial Clock Phase
 295                  * CTRLR0[ 9] Serial Clock Polarity
 296                  */
 297                 if (spi->mode & SPI_CPOL)
 298                         cr0 |= DW_HSSI_CTRLR0_SCPOL;
 299                 if (spi->mode & SPI_CPHA)
 300                         cr0 |= DW_HSSI_CTRLR0_SCPHA;
 301
 302                 /* CTRLR0[13] Shift Register Loop */
 303                 if (spi->mode & SPI_LOOP)
 304                         cr0 |= DW_HSSI_CTRLR0_SRL;
 305
 306                 /* CTRLR0[31] MST */
 307                 if (dw_spi_ver_is_ge(dws, HSSI, 102A))
 308                         cr0 |= DW_HSSI_CTRLR0_MST;
 309         }
 310
 311         return cr0;
 312 }
 313
 314 void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi,
 315                           struct dw_spi_cfg *cfg)
 316 {
 317         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 318         u32 cr0 = chip->cr0;
 319         u32 speed_hz;
 320         u16 clk_div;
 321
 322         /* CTRLR0[ 4/3: 0] or CTRLR0[ 20: 16] Data Frame Size */
 323         cr0 |= (cfg->dfs - 1) << dws->dfs_offset;
 324
 325         if (dw_spi_ip_is(dws, PSSI))
 326                 /* CTRLR0[ 9:8] Transfer Mode */
 327                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 328         else
 329                 /* CTRLR0[11:10] Transfer Mode */
 330                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 331
 332         dw_writel(dws, DW_SPI_CTRLR0, cr0);
 333
 334         if (cfg->tmode == DW_SPI_CTRLR0_TMOD_EPROMREAD ||
 335             cfg->tmode == DW_SPI_CTRLR0_TMOD_RO)
 336                 dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0);
 337
 338         /* Note DW APB SSI clock divider doesn't support odd numbers */
 339         clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe;
 340         speed_hz = dws->max_freq / clk_div;
 341
 342         if (dws->current_freq != speed_hz) {
 343                 dw_spi_set_clk(dws, clk_div);
 344                 dws->current_freq = speed_hz;
 345         }
 346
 347         /* Update RX sample delay if required */
 348         if (dws->cur_rx_sample_dly != chip->rx_sample_dly) {
 349                 dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly);
 350                 dws->cur_rx_sample_dly = chip->rx_sample_dly;
 351         }
 352 }
 353 EXPORT_SYMBOL_NS_GPL(dw_spi_update_config, SPI_DW_CORE);
 354
 355 static void dw_spi_irq_setup(struct dw_spi *dws)
 356 {
 357         u16 level;
 358         u8 imask;
 359
 360         /*
 361          * Originally Tx and Rx data lengths match. Rx FIFO Threshold level
 362          * will be adjusted at the final stage of the IRQ-based SPI transfer
 363          * execution so not to lose the leftover of the incoming data.
 364          */
 365         level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len);
 366         dw_writel(dws, DW_SPI_TXFTLR, level);
 367         dw_writel(dws, DW_SPI_RXFTLR, level - 1);
 368
 369         dws->transfer_handler = dw_spi_transfer_handler;
 370
 371         imask = DW_SPI_INT_TXEI | DW_SPI_INT_TXOI |
 372                 DW_SPI_INT_RXUI | DW_SPI_INT_RXOI | DW_SPI_INT_RXFI;
 373         dw_spi_umask_intr(dws, imask);
 374 }
 375
 376 /*
 377  * The iterative procedure of the poll-based transfer is simple: write as much
 378  * as possible to the Tx FIFO, wait until the pending to receive data is ready
 379  * to be read, read it from the Rx FIFO and check whether the performed
 380  * procedure has been successful.
 381  *
 382  * Note this method the same way as the IRQ-based transfer won't work well for
 383  * the SPI devices connected to the controller with native CS due to the
 384  * automatic CS assertion/de-assertion.
 385  */
 386 static int dw_spi_poll_transfer(struct dw_spi *dws,
 387                                 struct spi_transfer *transfer)
 388 {
 389         struct spi_delay delay;
 390         u16 nbits;
 391         int ret;
 392
 393         delay.unit = SPI_DELAY_UNIT_SCK;
 394         nbits = dws->n_bytes * BITS_PER_BYTE;
 395
 396         do {
 397                 dw_writer(dws);
 398
 399                 delay.value = nbits * (dws->rx_len - dws->tx_len);
 400                 spi_delay_exec(&delay, transfer);
 401
 402                 dw_reader(dws);
 403
 404                 ret = dw_spi_check_status(dws, true);
 405                 if (ret)
 406                         return ret;
 407         } while (dws->rx_len);
 408
 409         return 0;
 410 }
 411
 412 static int dw_spi_transfer_one(struct spi_controller *host,
 413                                struct spi_device *spi,
 414                                struct spi_transfer *transfer)
 415 {
 416         struct dw_spi *dws = spi_controller_get_devdata(host);
 417         struct dw_spi_cfg cfg = {
 418                 .tmode = DW_SPI_CTRLR0_TMOD_TR,
 419                 .dfs = transfer->bits_per_word,
 420                 .freq = transfer->speed_hz,
 421         };
 422         int ret;
 423
 424         dws->dma_mapped = 0;
 425         dws->n_bytes = roundup_pow_of_two(BITS_TO_BYTES(transfer->bits_per_word));
 426         dws->tx = (void *)transfer->tx_buf;
 427         dws->tx_len = transfer->len / dws->n_bytes;
 428         dws->rx = transfer->rx_buf;
 429         dws->rx_len = dws->tx_len;
 430
 431         /* Ensure the data above is visible for all CPUs */
 432         smp_mb();
 433
 434         dw_spi_enable_chip(dws, 0);
 435
 436         dw_spi_update_config(dws, spi, &cfg);
 437
 438         transfer->effective_speed_hz = dws->current_freq;
 439
 440         /* Check if current transfer is a DMA transaction */
 441         if (host->can_dma && host->can_dma(host, spi, transfer))
 442                 dws->dma_mapped = host->cur_msg_mapped;
 443
 444         /* For poll mode just disable all interrupts */
 445         dw_spi_mask_intr(dws, 0xff);
 446
 447         if (dws->dma_mapped) {
 448                 ret = dws->dma_ops->dma_setup(dws, transfer);
 449                 if (ret)
 450                         return ret;
 451         }
 452
 453         dw_spi_enable_chip(dws, 1);
 454
 455         if (dws->dma_mapped)
 456                 return dws->dma_ops->dma_transfer(dws, transfer);
 457         else if (dws->irq == IRQ_NOTCONNECTED)
 458                 return dw_spi_poll_transfer(dws, transfer);
 459
 460         dw_spi_irq_setup(dws);
 461
 462         return 1;
 463 }
 464
 465 static void dw_spi_handle_err(struct spi_controller *host,
 466                               struct spi_message *msg)
 467 {
 468         struct dw_spi *dws = spi_controller_get_devdata(host);
 469
 470         if (dws->dma_mapped)
 471                 dws->dma_ops->dma_stop(dws);
 472
 473         dw_spi_reset_chip(dws);
 474 }
 475
 476 static int dw_spi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 477 {
 478         if (op->data.dir == SPI_MEM_DATA_IN)
 479                 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1);
 480
 481         return 0;
 482 }
 483
 484 static bool dw_spi_supports_mem_op(struct spi_mem *mem,
 485                                    const struct spi_mem_op *op)
 486 {
 487         if (op->data.buswidth > 1 || op->addr.buswidth > 1 ||
 488             op->dummy.buswidth > 1 || op->cmd.buswidth > 1)
 489                 return false;
 490
 491         return spi_mem_default_supports_op(mem, op);
 492 }
 493
 494 static int dw_spi_init_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op)
 495 {
 496         unsigned int i, j, len;
 497         u8 *out;
 498
 499         /*
 500          * Calculate the total length of the EEPROM command transfer and
 501          * either use the pre-allocated buffer or create a temporary one.
 502          */
 503         len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
 504         if (op->data.dir == SPI_MEM_DATA_OUT)
 505                 len += op->data.nbytes;
 506
 507         if (len <= DW_SPI_BUF_SIZE) {
 508                 out = dws->buf;
 509         } else {
 510                 out = kzalloc(len, GFP_KERNEL);
 511                 if (!out)
 512                         return -ENOMEM;
 513         }
 514
 515         /*
 516          * Collect the operation code, address and dummy bytes into the single
 517          * buffer. If it's a transfer with data to be sent, also copy it into the
 518          * single buffer in order to speed the data transmission up.
 519          */
 520         for (i = 0; i < op->cmd.nbytes; ++i)
 521                 out[i] = DW_SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1);
 522         for (j = 0; j < op->addr.nbytes; ++i, ++j)
 523                 out[i] = DW_SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1);
 524         for (j = 0; j < op->dummy.nbytes; ++i, ++j)
 525                 out[i] = 0x0;
 526
 527         if (op->data.dir == SPI_MEM_DATA_OUT)
 528                 memcpy(&out[i], op->data.buf.out, op->data.nbytes);
 529
 530         dws->n_bytes = 1;
 531         dws->tx = out;
 532         dws->tx_len = len;
 533         if (op->data.dir == SPI_MEM_DATA_IN) {
 534                 dws->rx = op->data.buf.in;
 535                 dws->rx_len = op->data.nbytes;
 536         } else {
 537                 dws->rx = NULL;
 538                 dws->rx_len = 0;
 539         }
 540
 541         return 0;
 542 }
 543
 544 static void dw_spi_free_mem_buf(struct dw_spi *dws)
 545 {
 546         if (dws->tx != dws->buf)
 547                 kfree(dws->tx);
 548 }
 549
 550 static int dw_spi_write_then_read(struct dw_spi *dws, struct spi_device *spi)
 551 {
 552         u32 room, entries, sts;
 553         unsigned int len;
 554         u8 *buf;
 555
 556         /*
 557          * At initial stage we just pre-fill the Tx FIFO in with no rush,
 558          * since native CS hasn't been enabled yet and the automatic data
 559          * transmission won't start til we do that.
 560          */
 561         len = min(dws->fifo_len, dws->tx_len);
 562         buf = dws->tx;
 563         while (len--)
 564                 dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 565
 566         /*
 567          * After setting any bit in the SER register the transmission will
 568          * start automatically. We have to keep up with that procedure
 569          * otherwise the CS de-assertion will happen whereupon the memory
 570          * operation will be pre-terminated.
 571          */
 572         len = dws->tx_len - ((void *)buf - dws->tx);
 573         dw_spi_set_cs(spi, false);
 574         while (len) {
 575                 entries = readl_relaxed(dws->regs + DW_SPI_TXFLR);
 576                 if (!entries) {
 577                         dev_err(&dws->host->dev, "CS de-assertion on Tx\n");
 578                         return -EIO;
 579                 }
 580                 room = min(dws->fifo_len - entries, len);
 581                 for (; room; --room, --len)
 582                         dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 583         }
 584
 585         /*
 586          * Data fetching will start automatically if the EEPROM-read mode is
 587          * activated. We have to keep up with the incoming data pace to
 588          * prevent the Rx FIFO overflow causing the inbound data loss.
 589          */
 590         len = dws->rx_len;
 591         buf = dws->rx;
 592         while (len) {
 593                 entries = readl_relaxed(dws->regs + DW_SPI_RXFLR);
 594                 if (!entries) {
 595                         sts = readl_relaxed(dws->regs + DW_SPI_RISR);
 596                         if (sts & DW_SPI_INT_RXOI) {
 597                                 dev_err(&dws->host->dev, "FIFO overflow on Rx\n");
 598                                 return -EIO;
 599                         }
 600                         continue;
 601                 }
 602                 entries = min(entries, len);
 603                 for (; entries; --entries, --len)
 604                         *buf++ = dw_read_io_reg(dws, DW_SPI_DR);
 605         }
 606
 607         return 0;
 608 }
 609
 610 static inline bool dw_spi_ctlr_busy(struct dw_spi *dws)
 611 {
 612         return dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_BUSY;
 613 }
 614
 615 static int dw_spi_wait_mem_op_done(struct dw_spi *dws)
 616 {
 617         int retry = DW_SPI_WAIT_RETRIES;
 618         struct spi_delay delay;
 619         unsigned long ns, us;
 620         u32 nents;
 621
 622         nents = dw_readl(dws, DW_SPI_TXFLR);
 623         ns = NSEC_PER_SEC / dws->current_freq * nents;
 624         ns *= dws->n_bytes * BITS_PER_BYTE;
 625         if (ns <= NSEC_PER_USEC) {
 626                 delay.unit = SPI_DELAY_UNIT_NSECS;
 627                 delay.value = ns;
 628         } else {
 629                 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
 630                 delay.unit = SPI_DELAY_UNIT_USECS;
 631                 delay.value = clamp_val(us, 0, USHRT_MAX);
 632         }
 633
 634         while (dw_spi_ctlr_busy(dws) && retry--)
 635                 spi_delay_exec(&delay, NULL);
 636
 637         if (retry < 0) {
 638                 dev_err(&dws->host->dev, "Mem op hanged up\n");
 639                 return -EIO;
 640         }
 641
 642         return 0;
 643 }
 644
 645 static void dw_spi_stop_mem_op(struct dw_spi *dws, struct spi_device *spi)
 646 {
 647         dw_spi_enable_chip(dws, 0);
 648         dw_spi_set_cs(spi, true);
 649         dw_spi_enable_chip(dws, 1);
 650 }
 651
 652 /*
 653  * The SPI memory operation implementation below is the best choice for the
 654  * devices, which are selected by the native chip-select lane. It's
 655  * specifically developed to workaround the problem with automatic chip-select
 656  * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current
 657  * SPI-mem core calls exec_op() callback only if the GPIO-based CS is
 658  * unavailable.
 659  */
 660 static int dw_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 661 {
 662         struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller);
 663         struct dw_spi_cfg cfg;
 664         unsigned long flags;
 665         int ret;
 666
 667         /*
 668          * Collect the outbound data into a single buffer to speed the
 669          * transmission up at least on the initial stage.
 670          */
 671         ret = dw_spi_init_mem_buf(dws, op);
 672         if (ret)
 673                 return ret;
 674
 675         /*
 676          * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN
 677          * operation. Transmit-only mode is suitable for the rest of them.
 678          */
 679         cfg.dfs = 8;
 680         cfg.freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_mem_freq);
 681         if (op->data.dir == SPI_MEM_DATA_IN) {
 682                 cfg.tmode = DW_SPI_CTRLR0_TMOD_EPROMREAD;
 683                 cfg.ndf = op->data.nbytes;
 684         } else {
 685                 cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
 686         }
 687
 688         dw_spi_enable_chip(dws, 0);
 689
 690         dw_spi_update_config(dws, mem->spi, &cfg);
 691
 692         dw_spi_mask_intr(dws, 0xff);
 693
 694         dw_spi_enable_chip(dws, 1);
 695
 696         /*
 697          * DW APB SSI controller has very nasty peculiarities. First originally
 698          * (without any vendor-specific modifications) it doesn't provide a
 699          * direct way to set and clear the native chip-select signal. Instead
 700          * the controller asserts the CS lane if Tx FIFO isn't empty and a
 701          * transmission is going on, and automatically de-asserts it back to
 702          * the high level if the Tx FIFO doesn't have anything to be pushed
 703          * out. Due to that a multi-tasking or heavy IRQs activity might be
 704          * fatal, since the transfer procedure preemption may cause the Tx FIFO
 705          * getting empty and sudden CS de-assertion, which in the middle of the
 706          * transfer will most likely cause the data loss. Secondly the
 707          * EEPROM-read or Read-only DW SPI transfer modes imply the incoming
 708          * data being automatically pulled in into the Rx FIFO. So if the
 709          * driver software is late in fetching the data from the FIFO before
 710          * it's overflown, new incoming data will be lost. In order to make
 711          * sure the executed memory operations are CS-atomic and to prevent the
 712          * Rx FIFO overflow we have to disable the local interrupts so to block
 713          * any preemption during the subsequent IO operations.
 714          *
 715          * Note. At some circumstances disabling IRQs may not help to prevent
 716          * the problems described above. The CS de-assertion and Rx FIFO
 717          * overflow may still happen due to the relatively slow system bus or
 718          * CPU not working fast enough, so the write-then-read algo implemented
 719          * here just won't keep up with the SPI bus data transfer. Such
 720          * situation is highly platform specific and is supposed to be fixed by
 721          * manually restricting the SPI bus frequency using the
 722          * dws->max_mem_freq parameter.
 723          */
 724         local_irq_save(flags);
 725         preempt_disable();
 726
 727         ret = dw_spi_write_then_read(dws, mem->spi);
 728
 729         local_irq_restore(flags);
 730         preempt_enable();
 731
 732         /*
 733          * Wait for the operation being finished and check the controller
 734          * status only if there hasn't been any run-time error detected. In the
 735          * former case it's just pointless. In the later one to prevent an
 736          * additional error message printing since any hw error flag being set
 737          * would be due to an error detected on the data transfer.
 738          */
 739         if (!ret) {
 740                 ret = dw_spi_wait_mem_op_done(dws);
 741                 if (!ret)
 742                         ret = dw_spi_check_status(dws, true);
 743         }
 744
 745         dw_spi_stop_mem_op(dws, mem->spi);
 746
 747         dw_spi_free_mem_buf(dws);
 748
 749         return ret;
 750 }
 751
 752 /*
 753  * Initialize the default memory operations if a glue layer hasn't specified
 754  * custom ones. Direct mapping operations will be preserved anyway since DW SPI
 755  * controller doesn't have an embedded dirmap interface. Note the memory
 756  * operations implemented in this driver is the best choice only for the DW APB
 757  * SSI controller with standard native CS functionality. If a hardware vendor
 758  * has fixed the automatic CS assertion/de-assertion peculiarity, then it will
 759  * be safer to use the normal SPI-messages-based transfers implementation.
 760  */
 761 static void dw_spi_init_mem_ops(struct dw_spi *dws)
 762 {
 763         if (!dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) &&
 764             !dws->set_cs) {
 765                 dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size;
 766                 dws->mem_ops.supports_op = dw_spi_supports_mem_op;
 767                 dws->mem_ops.exec_op = dw_spi_exec_mem_op;
 768                 if (!dws->max_mem_freq)
 769                         dws->max_mem_freq = dws->max_freq;
 770         }
 771 }
 772
 773 /* This may be called twice for each spi dev */
 774 static int dw_spi_setup(struct spi_device *spi)
 775 {
 776         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 777         struct dw_spi_chip_data *chip;
 778
 779         /* Only alloc on first setup */
 780         chip = spi_get_ctldata(spi);
 781         if (!chip) {
 782                 struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 783                 u32 rx_sample_dly_ns;
 784
 785                 chip = kzalloc(sizeof(*chip), GFP_KERNEL);
 786                 if (!chip)
 787                         return -ENOMEM;
 788                 spi_set_ctldata(spi, chip);
 789                 /* Get specific / default rx-sample-delay */
 790                 if (device_property_read_u32(&spi->dev,
 791                                              "rx-sample-delay-ns",
 792                                              &rx_sample_dly_ns) != 0)
 793                         /* Use default controller value */
 794                         rx_sample_dly_ns = dws->def_rx_sample_dly_ns;
 795                 chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,
 796                                                         NSEC_PER_SEC /
 797                                                         dws->max_freq);
 798         }
 799
 800         /*
 801          * Update CR0 data each time the setup callback is invoked since
 802          * the device parameters could have been changed, for instance, by
 803          * the MMC SPI driver or something else.
 804          */
 805         chip->cr0 = dw_spi_prepare_cr0(dws, spi);
 806
 807         return 0;
 808 }
 809
 810 static void dw_spi_cleanup(struct spi_device *spi)
 811 {
 812         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 813
 814         kfree(chip);
 815         spi_set_ctldata(spi, NULL);
 816 }
 817
 818 /* Restart the controller, disable all interrupts, clean rx fifo */
 819 static void dw_spi_hw_init(struct device *dev, struct dw_spi *dws)
 820 {
 821         dw_spi_reset_chip(dws);
 822
 823         /*
 824          * Retrieve the Synopsys component version if it hasn't been specified
 825          * by the platform. CoreKit version ID is encoded as a 3-chars ASCII
 826          * code enclosed with '*' (typical for the most of Synopsys IP-cores).
 827          */
 828         if (!dws->ver) {
 829                 dws->ver = dw_readl(dws, DW_SPI_VERSION);
 830
 831                 dev_dbg(dev, "Synopsys DWC%sSSI v%c.%c%c\n",
 832                         dw_spi_ip_is(dws, PSSI) ? " APB " : " ",
 833                         DW_SPI_GET_BYTE(dws->ver, 3), DW_SPI_GET_BYTE(dws->ver, 2),
 834                         DW_SPI_GET_BYTE(dws->ver, 1));
 835         }
 836
 837         /*
 838          * Try to detect the number of native chip-selects if the platform
 839          * driver didn't set it up. There can be up to 16 lines configured.
 840          */
 841         if (!dws->num_cs) {
 842                 u32 ser;
 843
 844                 dw_writel(dws, DW_SPI_SER, 0xffff);
 845                 ser = dw_readl(dws, DW_SPI_SER);
 846                 dw_writel(dws, DW_SPI_SER, 0);
 847
 848                 dws->num_cs = hweight16(ser);
 849         }
 850
 851         /*
 852          * Try to detect the FIFO depth if not set by interface driver,
 853          * the depth could be from 2 to 256 from HW spec
 854          */
 855         if (!dws->fifo_len) {
 856                 u32 fifo;
 857
 858                 for (fifo = 1; fifo < 256; fifo++) {
 859                         dw_writel(dws, DW_SPI_TXFTLR, fifo);
 860                         if (fifo != dw_readl(dws, DW_SPI_TXFTLR))
 861                                 break;
 862                 }
 863                 dw_writel(dws, DW_SPI_TXFTLR, 0);
 864
 865                 dws->fifo_len = (fifo == 1) ? 0 : fifo;
 866                 dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len);
 867         }
 868
 869         /*
 870          * Detect CTRLR0.DFS field size and offset by testing the lowest bits
 871          * writability. Note DWC SSI controller also has the extended DFS, but
 872          * with zero offset.
 873          */
 874         if (dw_spi_ip_is(dws, PSSI)) {
 875                 u32 cr0, tmp = dw_readl(dws, DW_SPI_CTRLR0);
 876
 877                 dw_spi_enable_chip(dws, 0);
 878                 dw_writel(dws, DW_SPI_CTRLR0, 0xffffffff);
 879                 cr0 = dw_readl(dws, DW_SPI_CTRLR0);
 880                 dw_writel(dws, DW_SPI_CTRLR0, tmp);
 881                 dw_spi_enable_chip(dws, 1);
 882
 883                 if (!(cr0 & DW_PSSI_CTRLR0_DFS_MASK)) {
 884                         dws->caps |= DW_SPI_CAP_DFS32;
 885                         dws->dfs_offset = __bf_shf(DW_PSSI_CTRLR0_DFS32_MASK);
 886                         dev_dbg(dev, "Detected 32-bits max data frame size\n");
 887                 }
 888         } else {
 889                 dws->caps |= DW_SPI_CAP_DFS32;
 890         }
 891
 892         /* enable HW fixup for explicit CS deselect for Amazon's alpine chip */
 893         if (dws->caps & DW_SPI_CAP_CS_OVERRIDE)
 894                 dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF);
 895 }
 896
 897 int dw_spi_add_host(struct device *dev, struct dw_spi *dws)
 898 {
 899         struct spi_controller *host;
 900         int ret;
 901
 902         if (!dws)
 903                 return -EINVAL;
 904
 905         host = spi_alloc_host(dev, 0);
 906         if (!host)
 907                 return -ENOMEM;
 908
 909         device_set_node(&host->dev, dev_fwnode(dev));
 910
 911         dws->host = host;
 912         dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR);
 913
 914         spi_controller_set_devdata(host, dws);
 915
 916         /* Basic HW init */
 917         dw_spi_hw_init(dev, dws);
 918
 919         ret = request_irq(dws->irq, dw_spi_irq, IRQF_SHARED, dev_name(dev),
 920                           host);
 921         if (ret < 0 && ret != -ENOTCONN) {
 922                 dev_err(dev, "can not get IRQ\n");
 923                 goto err_free_host;
 924         }
 925
 926         dw_spi_init_mem_ops(dws);
 927
 928         host->use_gpio_descriptors = true;
 929         host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;
 930         if (dws->caps & DW_SPI_CAP_DFS32)
 931                 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
 932         else
 933                 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
 934         host->bus_num = dws->bus_num;
 935         host->num_chipselect = dws->num_cs;
 936         host->setup = dw_spi_setup;
 937         host->cleanup = dw_spi_cleanup;
 938         if (dws->set_cs)
 939                 host->set_cs = dws->set_cs;
 940         else
 941                 host->set_cs = dw_spi_set_cs;
 942         host->transfer_one = dw_spi_transfer_one;
 943         host->handle_err = dw_spi_handle_err;
 944         if (dws->mem_ops.exec_op)
 945                 host->mem_ops = &dws->mem_ops;
 946         host->max_speed_hz = dws->max_freq;
 947         host->flags = SPI_CONTROLLER_GPIO_SS;
 948         host->auto_runtime_pm = true;
 949
 950         /* Get default rx sample delay */
 951         device_property_read_u32(dev, "rx-sample-delay-ns",
 952                                  &dws->def_rx_sample_dly_ns);
 953
 954         if (dws->dma_ops && dws->dma_ops->dma_init) {
 955                 ret = dws->dma_ops->dma_init(dev, dws);
 956                 if (ret == -EPROBE_DEFER) {
 957                         goto err_free_irq;
 958                 } else if (ret) {
 959                         dev_warn(dev, "DMA init failed\n");
 960                 } else {
 961                         host->can_dma = dws->dma_ops->can_dma;
 962                         host->flags |= SPI_CONTROLLER_MUST_TX;
 963                 }
 964         }
 965
 966         ret = spi_register_controller(host);
 967         if (ret) {
 968                 dev_err_probe(dev, ret, "problem registering spi host\n");
 969                 goto err_dma_exit;
 970         }
 971
 972         dw_spi_debugfs_init(dws);
 973         return 0;
 974
 975 err_dma_exit:
 976         if (dws->dma_ops && dws->dma_ops->dma_exit)
 977                 dws->dma_ops->dma_exit(dws);
 978         dw_spi_enable_chip(dws, 0);
 979 err_free_irq:
 980         free_irq(dws->irq, host);
 981 err_free_host:
 982         spi_controller_put(host);
 983         return ret;
 984 }
 985 EXPORT_SYMBOL_NS_GPL(dw_spi_add_host, SPI_DW_CORE);
 986
 987 void dw_spi_remove_host(struct dw_spi *dws)
 988 {
 989         dw_spi_debugfs_remove(dws);
 990
 991         spi_unregister_controller(dws->host);
 992
 993         if (dws->dma_ops && dws->dma_ops->dma_exit)
 994                 dws->dma_ops->dma_exit(dws);
 995
 996         dw_spi_shutdown_chip(dws);
 997
 998         free_irq(dws->irq, dws->host);
 999 }
1000 EXPORT_SYMBOL_NS_GPL(dw_spi_remove_host, SPI_DW_CORE);
1001
1002 int dw_spi_suspend_host(struct dw_spi *dws)
1003 {
1004         int ret;
1005
1006         ret = spi_controller_suspend(dws->host);
1007         if (ret)
1008                 return ret;
1009
1010         dw_spi_shutdown_chip(dws);
1011         return 0;
1012 }
1013 EXPORT_SYMBOL_NS_GPL(dw_spi_suspend_host, SPI_DW_CORE);
1014
1015 int dw_spi_resume_host(struct dw_spi *dws)
1016 {
1017         dw_spi_hw_init(&dws->host->dev, dws);
1018         return spi_controller_resume(dws->host);
1019 }
1020 EXPORT_SYMBOL_NS_GPL(dw_spi_resume_host, SPI_DW_CORE);
1021
1022 MODULE_AUTHOR("Feng Tang <[email protected]>");
1023 MODULE_DESCRIPTION("Driver for DesignWare SPI controller core");
1024 MODULE_LICENSE("GPL v2");