drivers/mtd/nand/raw/sunxi_nand.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2013 Boris BREZILLON <[email protected]>
   4  *
   5  * Derived from:
   6  *      https://github.com/yuq/sunxi-nfc-mtd
   7  *      Copyright (C) 2013 Qiang Yu <[email protected]>
   8  *
   9  *      https://github.com/hno/Allwinner-Info
  10  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  11  *
  12  *      Copyright (C) 2013 Dmitriy B. <[email protected]>
  13  *      Copyright (C) 2013 Sergey Lapin <[email protected]>
  14  */
  15
  16 #include <linux/dma-mapping.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/of.h>
  22 #include <linux/mtd/mtd.h>
  23 #include <linux/mtd/rawnand.h>
  24 #include <linux/mtd/partitions.h>
  25 #include <linux/clk.h>
  26 #include <linux/delay.h>
  27 #include <linux/dmaengine.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/reset.h>
  31
  32 #define NFC_REG_CTL             0x0000
  33 #define NFC_REG_ST              0x0004
  34 #define NFC_REG_INT             0x0008
  35 #define NFC_REG_TIMING_CTL      0x000C
  36 #define NFC_REG_TIMING_CFG      0x0010
  37 #define NFC_REG_ADDR_LOW        0x0014
  38 #define NFC_REG_ADDR_HIGH       0x0018
  39 #define NFC_REG_SECTOR_NUM      0x001C
  40 #define NFC_REG_CNT             0x0020
  41 #define NFC_REG_CMD             0x0024
  42 #define NFC_REG_RCMD_SET        0x0028
  43 #define NFC_REG_WCMD_SET        0x002C
  44 #define NFC_REG_A10_IO_DATA     0x0030
  45 #define NFC_REG_A23_IO_DATA     0x0300
  46 #define NFC_REG_ECC_CTL         0x0034
  47 #define NFC_REG_ECC_ST          0x0038
  48 #define NFC_REG_DEBUG           0x003C
  49 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  50 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  51 #define NFC_REG_SPARE_AREA      0x00A0
  52 #define NFC_REG_PAT_ID          0x00A4
  53 #define NFC_REG_MDMA_ADDR       0x00C0
  54 #define NFC_REG_MDMA_CNT        0x00C4
  55 #define NFC_RAM0_BASE           0x0400
  56 #define NFC_RAM1_BASE           0x0800
  57
  58 /* define bit use in NFC_CTL */
  59 #define NFC_EN                  BIT(0)
  60 #define NFC_RESET               BIT(1)
  61 #define NFC_BUS_WIDTH_MSK       BIT(2)
  62 #define NFC_BUS_WIDTH_8         (0 << 2)
  63 #define NFC_BUS_WIDTH_16        (1 << 2)
  64 #define NFC_RB_SEL_MSK          BIT(3)
  65 #define NFC_RB_SEL(x)           ((x) << 3)
  66 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  67 #define NFC_CE_SEL(x)           ((x) << 24)
  68 #define NFC_CE_CTL              BIT(6)
  69 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  70 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  71 #define NFC_SAM                 BIT(12)
  72 #define NFC_RAM_METHOD          BIT(14)
  73 #define NFC_DMA_TYPE_NORMAL     BIT(15)
  74 #define NFC_DEBUG_CTL           BIT(31)
  75
  76 /* define bit use in NFC_ST */
  77 #define NFC_RB_B2R              BIT(0)
  78 #define NFC_CMD_INT_FLAG        BIT(1)
  79 #define NFC_DMA_INT_FLAG        BIT(2)
  80 #define NFC_CMD_FIFO_STATUS     BIT(3)
  81 #define NFC_STA                 BIT(4)
  82 #define NFC_NATCH_INT_FLAG      BIT(5)
  83 #define NFC_RB_STATE(x)         BIT(x + 8)
  84
  85 /* define bit use in NFC_INT */
  86 #define NFC_B2R_INT_ENABLE      BIT(0)
  87 #define NFC_CMD_INT_ENABLE      BIT(1)
  88 #define NFC_DMA_INT_ENABLE      BIT(2)
  89 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  90                                  NFC_CMD_INT_ENABLE | \
  91                                  NFC_DMA_INT_ENABLE)
  92
  93 /* define bit use in NFC_TIMING_CTL */
  94 #define NFC_TIMING_CTL_EDO      BIT(8)
  95
  96 /* define NFC_TIMING_CFG register layout */
  97 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
  98         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
  99         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 100         (((tCAD) & 0x7) << 8))
 101
 102 /* define bit use in NFC_CMD */
 103 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 104 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 105 #define NFC_CMD(x)              (x)
 106 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 107 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 108 #define NFC_SEND_ADR            BIT(19)
 109 #define NFC_ACCESS_DIR          BIT(20)
 110 #define NFC_DATA_TRANS          BIT(21)
 111 #define NFC_SEND_CMD1           BIT(22)
 112 #define NFC_WAIT_FLAG           BIT(23)
 113 #define NFC_SEND_CMD2           BIT(24)
 114 #define NFC_SEQ                 BIT(25)
 115 #define NFC_DATA_SWAP_METHOD    BIT(26)
 116 #define NFC_ROW_AUTO_INC        BIT(27)
 117 #define NFC_SEND_CMD3           BIT(28)
 118 #define NFC_SEND_CMD4           BIT(29)
 119 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 120 #define NFC_NORMAL_OP           (0 << 30)
 121 #define NFC_ECC_OP              (1 << 30)
 122 #define NFC_PAGE_OP             (2U << 30)
 123
 124 /* define bit use in NFC_RCMD_SET */
 125 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 126 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 127 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 128
 129 /* define bit use in NFC_WCMD_SET */
 130 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 131 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 132 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 133 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 134
 135 /* define bit use in NFC_ECC_CTL */
 136 #define NFC_ECC_EN              BIT(0)
 137 #define NFC_ECC_PIPELINE        BIT(3)
 138 #define NFC_ECC_EXCEPTION       BIT(4)
 139 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 140 #define NFC_ECC_BLOCK_512       BIT(5)
 141 #define NFC_RANDOM_EN           BIT(9)
 142 #define NFC_RANDOM_DIRECTION    BIT(10)
 143 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 144 #define NFC_ECC_MODE(x)         ((x) << 12)
 145 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 146 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 147
 148 /* define bit use in NFC_ECC_ST */
 149 #define NFC_ECC_ERR(x)          BIT(x)
 150 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 151 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 152 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 153
 154 #define NFC_DEFAULT_TIMEOUT_MS  1000
 155
 156 #define NFC_SRAM_SIZE           1024
 157
 158 #define NFC_MAX_CS              7
 159
 160 /**
 161  * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
 162  *
 163  * @cs: the NAND CS id used to communicate with a NAND Chip
 164  * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
 165  */
 166 struct sunxi_nand_chip_sel {
 167         u8 cs;
 168         s8 rb;
 169 };
 170
 171 /**
 172  * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
 173  *
 174  * @ecc_ctl: ECC_CTL register value for this NAND chip
 175  */
 176 struct sunxi_nand_hw_ecc {
 177         u32 ecc_ctl;
 178 };
 179
 180 /**
 181  * struct sunxi_nand_chip - stores NAND chip device related information
 182  *
 183  * @node: used to store NAND chips into a list
 184  * @nand: base NAND chip structure
 185  * @ecc: ECC controller structure
 186  * @clk_rate: clk_rate required for this NAND chip
 187  * @timing_cfg: TIMING_CFG register value for this NAND chip
 188  * @timing_ctl: TIMING_CTL register value for this NAND chip
 189  * @nsels: number of CS lines required by the NAND chip
 190  * @sels: array of CS lines descriptions
 191  */
 192 struct sunxi_nand_chip {
 193         struct list_head node;
 194         struct nand_chip nand;
 195         struct sunxi_nand_hw_ecc ecc;
 196         unsigned long clk_rate;
 197         u32 timing_cfg;
 198         u32 timing_ctl;
 199         int nsels;
 200         struct sunxi_nand_chip_sel sels[] __counted_by(nsels);
 201 };
 202
 203 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 204 {
 205         return container_of(nand, struct sunxi_nand_chip, nand);
 206 }
 207
 208 /*
 209  * NAND Controller capabilities structure: stores NAND controller capabilities
 210  * for distinction between compatible strings.
 211  *
 212  * @has_mdma:           Use mbus dma mode, otherwise general dma
 213  *                      through MBUS on A23/A33 needs extra configuration.
 214  * @reg_io_data:        I/O data register
 215  * @dma_maxburst:       DMA maxburst
 216  */
 217 struct sunxi_nfc_caps {
 218         bool has_mdma;
 219         unsigned int reg_io_data;
 220         unsigned int dma_maxburst;
 221 };
 222
 223 /**
 224  * struct sunxi_nfc - stores sunxi NAND controller information
 225  *
 226  * @controller: base controller structure
 227  * @dev: parent device (used to print error messages)
 228  * @regs: NAND controller registers
 229  * @ahb_clk: NAND controller AHB clock
 230  * @mod_clk: NAND controller mod clock
 231  * @reset: NAND controller reset line
 232  * @assigned_cs: bitmask describing already assigned CS lines
 233  * @clk_rate: NAND controller current clock rate
 234  * @chips: a list containing all the NAND chips attached to this NAND
 235  *         controller
 236  * @complete: a completion object used to wait for NAND controller events
 237  * @dmac: the DMA channel attached to the NAND controller
 238  * @caps: NAND Controller capabilities
 239  */
 240 struct sunxi_nfc {
 241         struct nand_controller controller;
 242         struct device *dev;
 243         void __iomem *regs;
 244         struct clk *ahb_clk;
 245         struct clk *mod_clk;
 246         struct reset_control *reset;
 247         unsigned long assigned_cs;
 248         unsigned long clk_rate;
 249         struct list_head chips;
 250         struct completion complete;
 251         struct dma_chan *dmac;
 252         const struct sunxi_nfc_caps *caps;
 253 };
 254
 255 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 256 {
 257         return container_of(ctrl, struct sunxi_nfc, controller);
 258 }
 259
 260 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 261 {
 262         struct sunxi_nfc *nfc = dev_id;
 263         u32 st = readl(nfc->regs + NFC_REG_ST);
 264         u32 ien = readl(nfc->regs + NFC_REG_INT);
 265
 266         if (!(ien & st))
 267                 return IRQ_NONE;
 268
 269         if ((ien & st) == ien)
 270                 complete(&nfc->complete);
 271
 272         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 273         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 274
 275         return IRQ_HANDLED;
 276 }
 277
 278 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 279                                  bool use_polling, unsigned int timeout_ms)
 280 {
 281         int ret;
 282
 283         if (events & ~NFC_INT_MASK)
 284                 return -EINVAL;
 285
 286         if (!timeout_ms)
 287                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 288
 289         if (!use_polling) {
 290                 init_completion(&nfc->complete);
 291
 292                 writel(events, nfc->regs + NFC_REG_INT);
 293
 294                 ret = wait_for_completion_timeout(&nfc->complete,
 295                                                 msecs_to_jiffies(timeout_ms));
 296                 if (!ret)
 297                         ret = -ETIMEDOUT;
 298                 else
 299                         ret = 0;
 300
 301                 writel(0, nfc->regs + NFC_REG_INT);
 302         } else {
 303                 u32 status;
 304
 305                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 306                                          (status & events) == events, 1,
 307                                          timeout_ms * 1000);
 308         }
 309
 310         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 311
 312         if (ret)
 313                 dev_err(nfc->dev, "wait interrupt timedout\n");
 314
 315         return ret;
 316 }
 317
 318 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 319 {
 320         u32 status;
 321         int ret;
 322
 323         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 324                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 325                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 326         if (ret)
 327                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 328
 329         return ret;
 330 }
 331
 332 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 333 {
 334         u32 ctl;
 335         int ret;
 336
 337         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 338         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 339
 340         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 341                                  !(ctl & NFC_RESET), 1,
 342                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 343         if (ret)
 344                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 345
 346         return ret;
 347 }
 348
 349 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
 350                                     int chunksize, int nchunks,
 351                                     enum dma_data_direction ddir,
 352                                     struct scatterlist *sg)
 353 {
 354         struct dma_async_tx_descriptor *dmad;
 355         enum dma_transfer_direction tdir;
 356         dma_cookie_t dmat;
 357         int ret;
 358
 359         if (ddir == DMA_FROM_DEVICE)
 360                 tdir = DMA_DEV_TO_MEM;
 361         else
 362                 tdir = DMA_MEM_TO_DEV;
 363
 364         sg_init_one(sg, buf, nchunks * chunksize);
 365         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 366         if (!ret)
 367                 return -ENOMEM;
 368
 369         if (!nfc->caps->has_mdma) {
 370                 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 371                 if (!dmad) {
 372                         ret = -EINVAL;
 373                         goto err_unmap_buf;
 374                 }
 375         }
 376
 377         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 378                nfc->regs + NFC_REG_CTL);
 379         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 380         writel(chunksize, nfc->regs + NFC_REG_CNT);
 381
 382         if (nfc->caps->has_mdma) {
 383                 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,
 384                        nfc->regs + NFC_REG_CTL);
 385                 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT);
 386                 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR);
 387         } else {
 388                 dmat = dmaengine_submit(dmad);
 389
 390                 ret = dma_submit_error(dmat);
 391                 if (ret)
 392                         goto err_clr_dma_flag;
 393         }
 394
 395         return 0;
 396
 397 err_clr_dma_flag:
 398         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 399                nfc->regs + NFC_REG_CTL);
 400
 401 err_unmap_buf:
 402         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 403         return ret;
 404 }
 405
 406 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
 407                                      enum dma_data_direction ddir,
 408                                      struct scatterlist *sg)
 409 {
 410         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 411         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 412                nfc->regs + NFC_REG_CTL);
 413 }
 414
 415 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
 416 {
 417         struct mtd_info *mtd = nand_to_mtd(nand);
 418         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 419         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 420         struct sunxi_nand_chip_sel *sel;
 421         u32 ctl;
 422
 423         if (cs >= sunxi_nand->nsels)
 424                 return;
 425
 426         ctl = readl(nfc->regs + NFC_REG_CTL) &
 427               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 428
 429         sel = &sunxi_nand->sels[cs];
 430         ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
 431         if (sel->rb >= 0)
 432                 ctl |= NFC_RB_SEL(sel->rb);
 433
 434         writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 435
 436         if (nfc->clk_rate != sunxi_nand->clk_rate) {
 437                 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 438                 nfc->clk_rate = sunxi_nand->clk_rate;
 439         }
 440
 441         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 442         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 443         writel(ctl, nfc->regs + NFC_REG_CTL);
 444 }
 445
 446 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 447 {
 448         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 449         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 450         int ret;
 451         int cnt;
 452         int offs = 0;
 453         u32 tmp;
 454
 455         while (len > offs) {
 456                 bool poll = false;
 457
 458                 cnt = min(len - offs, NFC_SRAM_SIZE);
 459
 460                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 461                 if (ret)
 462                         break;
 463
 464                 writel(cnt, nfc->regs + NFC_REG_CNT);
 465                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 466                 writel(tmp, nfc->regs + NFC_REG_CMD);
 467
 468                 /* Arbitrary limit for polling mode */
 469                 if (cnt < 64)
 470                         poll = true;
 471
 472                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 473                 if (ret)
 474                         break;
 475
 476                 if (buf)
 477                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 478                                       cnt);
 479                 offs += cnt;
 480         }
 481 }
 482
 483 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 484                                 int len)
 485 {
 486         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 487         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 488         int ret;
 489         int cnt;
 490         int offs = 0;
 491         u32 tmp;
 492
 493         while (len > offs) {
 494                 bool poll = false;
 495
 496                 cnt = min(len - offs, NFC_SRAM_SIZE);
 497
 498                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 499                 if (ret)
 500                         break;
 501
 502                 writel(cnt, nfc->regs + NFC_REG_CNT);
 503                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 504                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 505                       NFC_ACCESS_DIR;
 506                 writel(tmp, nfc->regs + NFC_REG_CMD);
 507
 508                 /* Arbitrary limit for polling mode */
 509                 if (cnt < 64)
 510                         poll = true;
 511
 512                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 513                 if (ret)
 514                         break;
 515
 516                 offs += cnt;
 517         }
 518 }
 519
 520 /* These seed values have been extracted from Allwinner's BSP */
 521 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 522         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 523         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 524         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 525         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 526         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 527         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 528         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 529         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 530         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 531         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 532         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 533         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 534         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 535         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 536         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 537         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 538 };
 539
 540 /*
 541  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 542  * have been generated using
 543  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 544  * the randomizer engine does internally before de/scrambling OOB data.
 545  *
 546  * Those tables are statically defined to avoid calculating randomizer state
 547  * at runtime.
 548  */
 549 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 550         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 551         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 552         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 553         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 554         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 555         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 556         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 557         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 558         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 559         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 560         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 561         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 562         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 563         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 564         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 565         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 566 };
 567
 568 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 569         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 570         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 571         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 572         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 573         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 574         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 575         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 576         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 577         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 578         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 579         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 580         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 581         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 582         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 583         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 584         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 585 };
 586
 587 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 588 {
 589         state &= 0x7fff;
 590
 591         /*
 592          * This loop is just a simple implementation of a Fibonacci LFSR using
 593          * the x16 + x15 + 1 polynomial.
 594          */
 595         while (count--)
 596                 state = ((state >> 1) |
 597                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 598
 599         return state;
 600 }
 601
 602 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
 603                                       bool ecc)
 604 {
 605         struct mtd_info *mtd = nand_to_mtd(nand);
 606         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 607         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 608
 609         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 610                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 611
 612         if (ecc) {
 613                 if (mtd->ecc_step_size == 512)
 614                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 615                 else
 616                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 617         }
 618
 619         return seeds[page % mod];
 620 }
 621
 622 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
 623                                         bool ecc)
 624 {
 625         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 626         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 627         u16 state;
 628
 629         if (!(nand->options & NAND_NEED_SCRAMBLING))
 630                 return;
 631
 632         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 633         state = sunxi_nfc_randomizer_state(nand, page, ecc);
 634         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 635         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 636 }
 637
 638 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
 639 {
 640         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 641
 642         if (!(nand->options & NAND_NEED_SCRAMBLING))
 643                 return;
 644
 645         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 646                nfc->regs + NFC_REG_ECC_CTL);
 647 }
 648
 649 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
 650 {
 651         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 652
 653         if (!(nand->options & NAND_NEED_SCRAMBLING))
 654                 return;
 655
 656         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 657                nfc->regs + NFC_REG_ECC_CTL);
 658 }
 659
 660 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
 661 {
 662         u16 state = sunxi_nfc_randomizer_state(nand, page, true);
 663
 664         bbm[0] ^= state;
 665         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 666 }
 667
 668 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
 669                                            const uint8_t *buf, int len,
 670                                            bool ecc, int page)
 671 {
 672         sunxi_nfc_randomizer_config(nand, page, ecc);
 673         sunxi_nfc_randomizer_enable(nand);
 674         sunxi_nfc_write_buf(nand, buf, len);
 675         sunxi_nfc_randomizer_disable(nand);
 676 }
 677
 678 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
 679                                           int len, bool ecc, int page)
 680 {
 681         sunxi_nfc_randomizer_config(nand, page, ecc);
 682         sunxi_nfc_randomizer_enable(nand);
 683         sunxi_nfc_read_buf(nand, buf, len);
 684         sunxi_nfc_randomizer_disable(nand);
 685 }
 686
 687 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
 688 {
 689         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 690         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 691
 692         writel(sunxi_nand->ecc.ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 693 }
 694
 695 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
 696 {
 697         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 698
 699         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 700 }
 701
 702 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 703 {
 704         buf[0] = user_data;
 705         buf[1] = user_data >> 8;
 706         buf[2] = user_data >> 16;
 707         buf[3] = user_data >> 24;
 708 }
 709
 710 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 711 {
 712         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 713 }
 714
 715 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
 716                                                 int step, bool bbm, int page)
 717 {
 718         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 719
 720         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 721                                    oob);
 722
 723         /* De-randomize the Bad Block Marker. */
 724         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 725                 sunxi_nfc_randomize_bbm(nand, page, oob);
 726 }
 727
 728 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 729                                                 const u8 *oob, int step,
 730                                                 bool bbm, int page)
 731 {
 732         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 733         u8 user_data[4];
 734
 735         /* Randomize the Bad Block Marker. */
 736         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 737                 memcpy(user_data, oob, sizeof(user_data));
 738                 sunxi_nfc_randomize_bbm(nand, page, user_data);
 739                 oob = user_data;
 740         }
 741
 742         writel(sunxi_nfc_buf_to_user_data(oob),
 743                nfc->regs + NFC_REG_USER_DATA(step));
 744 }
 745
 746 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
 747                                           unsigned int *max_bitflips, int ret)
 748 {
 749         struct mtd_info *mtd = nand_to_mtd(nand);
 750
 751         if (ret < 0) {
 752                 mtd->ecc_stats.failed++;
 753         } else {
 754                 mtd->ecc_stats.corrected += ret;
 755                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 756         }
 757 }
 758
 759 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
 760                                     int step, u32 status, bool *erased)
 761 {
 762         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 763         struct nand_ecc_ctrl *ecc = &nand->ecc;
 764         u32 tmp;
 765
 766         *erased = false;
 767
 768         if (status & NFC_ECC_ERR(step))
 769                 return -EBADMSG;
 770
 771         if (status & NFC_ECC_PAT_FOUND(step)) {
 772                 u8 pattern;
 773
 774                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 775                         pattern = 0x0;
 776                 } else {
 777                         pattern = 0xff;
 778                         *erased = true;
 779                 }
 780
 781                 if (data)
 782                         memset(data, pattern, ecc->size);
 783
 784                 if (oob)
 785                         memset(oob, pattern, ecc->bytes + 4);
 786
 787                 return 0;
 788         }
 789
 790         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 791
 792         return NFC_ECC_ERR_CNT(step, tmp);
 793 }
 794
 795 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
 796                                        u8 *data, int data_off,
 797                                        u8 *oob, int oob_off,
 798                                        int *cur_off,
 799                                        unsigned int *max_bitflips,
 800                                        bool bbm, bool oob_required, int page)
 801 {
 802         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 803         struct nand_ecc_ctrl *ecc = &nand->ecc;
 804         int raw_mode = 0;
 805         bool erased;
 806         int ret;
 807
 808         if (*cur_off != data_off)
 809                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 810
 811         sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
 812
 813         if (data_off + ecc->size != oob_off)
 814                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 815
 816         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 817         if (ret)
 818                 return ret;
 819
 820         sunxi_nfc_randomizer_enable(nand);
 821         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 822                nfc->regs + NFC_REG_CMD);
 823
 824         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 825         sunxi_nfc_randomizer_disable(nand);
 826         if (ret)
 827                 return ret;
 828
 829         *cur_off = oob_off + ecc->bytes + 4;
 830
 831         ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
 832                                        readl(nfc->regs + NFC_REG_ECC_ST),
 833                                        &erased);
 834         if (erased)
 835                 return 1;
 836
 837         if (ret < 0) {
 838                 /*
 839                  * Re-read the data with the randomizer disabled to identify
 840                  * bitflips in erased pages.
 841                  */
 842                 if (nand->options & NAND_NEED_SCRAMBLING)
 843                         nand_change_read_column_op(nand, data_off, data,
 844                                                    ecc->size, false);
 845                 else
 846                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 847                                       ecc->size);
 848
 849                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 850                                            false);
 851
 852                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 853                                                   oob, ecc->bytes + 4,
 854                                                   NULL, 0, ecc->strength);
 855                 if (ret >= 0)
 856                         raw_mode = 1;
 857         } else {
 858                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 859
 860                 if (oob_required) {
 861                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 862                                                    false);
 863                         sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
 864                                                       true, page);
 865
 866                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
 867                                                             bbm, page);
 868                 }
 869         }
 870
 871         sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
 872
 873         return raw_mode;
 874 }
 875
 876 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
 877                                             u8 *oob, int *cur_off,
 878                                             bool randomize, int page)
 879 {
 880         struct mtd_info *mtd = nand_to_mtd(nand);
 881         struct nand_ecc_ctrl *ecc = &nand->ecc;
 882         int offset = ((ecc->bytes + 4) * ecc->steps);
 883         int len = mtd->oobsize - offset;
 884
 885         if (len <= 0)
 886                 return;
 887
 888         if (!cur_off || *cur_off != offset)
 889                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 890                                            false);
 891
 892         if (!randomize)
 893                 sunxi_nfc_read_buf(nand, oob + offset, len);
 894         else
 895                 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
 896                                               false, page);
 897
 898         if (cur_off)
 899                 *cur_off = mtd->oobsize + mtd->writesize;
 900 }
 901
 902 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
 903                                             int oob_required, int page,
 904                                             int nchunks)
 905 {
 906         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
 907         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 908         struct mtd_info *mtd = nand_to_mtd(nand);
 909         struct nand_ecc_ctrl *ecc = &nand->ecc;
 910         unsigned int max_bitflips = 0;
 911         int ret, i, raw_mode = 0;
 912         struct scatterlist sg;
 913         u32 status, wait;
 914
 915         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 916         if (ret)
 917                 return ret;
 918
 919         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
 920                                        DMA_FROM_DEVICE, &sg);
 921         if (ret)
 922                 return ret;
 923
 924         sunxi_nfc_hw_ecc_enable(nand);
 925         sunxi_nfc_randomizer_config(nand, page, false);
 926         sunxi_nfc_randomizer_enable(nand);
 927
 928         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
 929                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
 930
 931         wait = NFC_CMD_INT_FLAG;
 932
 933         if (nfc->caps->has_mdma)
 934                 wait |= NFC_DMA_INT_FLAG;
 935         else
 936                 dma_async_issue_pending(nfc->dmac);
 937
 938         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
 939                nfc->regs + NFC_REG_CMD);
 940
 941         ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
 942         if (ret && !nfc->caps->has_mdma)
 943                 dmaengine_terminate_all(nfc->dmac);
 944
 945         sunxi_nfc_randomizer_disable(nand);
 946         sunxi_nfc_hw_ecc_disable(nand);
 947
 948         sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
 949
 950         if (ret)
 951                 return ret;
 952
 953         status = readl(nfc->regs + NFC_REG_ECC_ST);
 954
 955         for (i = 0; i < nchunks; i++) {
 956                 int data_off = i * ecc->size;
 957                 int oob_off = i * (ecc->bytes + 4);
 958                 u8 *data = buf + data_off;
 959                 u8 *oob = nand->oob_poi + oob_off;
 960                 bool erased;
 961
 962                 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
 963                                                oob_required ? oob : NULL,
 964                                                i, status, &erased);
 965
 966                 /* ECC errors are handled in the second loop. */
 967                 if (ret < 0)
 968                         continue;
 969
 970                 if (oob_required && !erased) {
 971                         /* TODO: use DMA to retrieve OOB */
 972                         nand_change_read_column_op(nand,
 973                                                    mtd->writesize + oob_off,
 974                                                    oob, ecc->bytes + 4, false);
 975
 976                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
 977                                                             !i, page);
 978                 }
 979
 980                 if (erased)
 981                         raw_mode = 1;
 982
 983                 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 984         }
 985
 986         if (status & NFC_ECC_ERR_MSK) {
 987                 for (i = 0; i < nchunks; i++) {
 988                         int data_off = i * ecc->size;
 989                         int oob_off = i * (ecc->bytes + 4);
 990                         u8 *data = buf + data_off;
 991                         u8 *oob = nand->oob_poi + oob_off;
 992
 993                         if (!(status & NFC_ECC_ERR(i)))
 994                                 continue;
 995
 996                         /*
 997                          * Re-read the data with the randomizer disabled to
 998                          * identify bitflips in erased pages.
 999                          * TODO: use DMA to read page in raw mode
1000                          */
1001                         if (randomized)
1002                                 nand_change_read_column_op(nand, data_off,
1003                                                            data, ecc->size,
1004                                                            false);
1005
1006                         /* TODO: use DMA to retrieve OOB */
1007                         nand_change_read_column_op(nand,
1008                                                    mtd->writesize + oob_off,
1009                                                    oob, ecc->bytes + 4, false);
1010
1011                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1012                                                           oob, ecc->bytes + 4,
1013                                                           NULL, 0,
1014                                                           ecc->strength);
1015                         if (ret >= 0)
1016                                 raw_mode = 1;
1017
1018                         sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1019                 }
1020         }
1021
1022         if (oob_required)
1023                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1024                                                 NULL, !raw_mode,
1025                                                 page);
1026
1027         return max_bitflips;
1028 }
1029
1030 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1031                                         const u8 *data, int data_off,
1032                                         const u8 *oob, int oob_off,
1033                                         int *cur_off, bool bbm,
1034                                         int page)
1035 {
1036         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1037         struct nand_ecc_ctrl *ecc = &nand->ecc;
1038         int ret;
1039
1040         if (data_off != *cur_off)
1041                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1042
1043         sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1044
1045         if (data_off + ecc->size != oob_off)
1046                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1047
1048         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1049         if (ret)
1050                 return ret;
1051
1052         sunxi_nfc_randomizer_enable(nand);
1053         sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1054
1055         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1056                NFC_ACCESS_DIR | NFC_ECC_OP,
1057                nfc->regs + NFC_REG_CMD);
1058
1059         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1060         sunxi_nfc_randomizer_disable(nand);
1061         if (ret)
1062                 return ret;
1063
1064         *cur_off = oob_off + ecc->bytes + 4;
1065
1066         return 0;
1067 }
1068
1069 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1070                                              u8 *oob, int *cur_off,
1071                                              int page)
1072 {
1073         struct mtd_info *mtd = nand_to_mtd(nand);
1074         struct nand_ecc_ctrl *ecc = &nand->ecc;
1075         int offset = ((ecc->bytes + 4) * ecc->steps);
1076         int len = mtd->oobsize - offset;
1077
1078         if (len <= 0)
1079                 return;
1080
1081         if (!cur_off || *cur_off != offset)
1082                 nand_change_write_column_op(nand, offset + mtd->writesize,
1083                                             NULL, 0, false);
1084
1085         sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1086
1087         if (cur_off)
1088                 *cur_off = mtd->oobsize + mtd->writesize;
1089 }
1090
1091 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1092                                       int oob_required, int page)
1093 {
1094         struct mtd_info *mtd = nand_to_mtd(nand);
1095         struct nand_ecc_ctrl *ecc = &nand->ecc;
1096         unsigned int max_bitflips = 0;
1097         int ret, i, cur_off = 0;
1098         bool raw_mode = false;
1099
1100         sunxi_nfc_select_chip(nand, nand->cur_cs);
1101
1102         nand_read_page_op(nand, page, 0, NULL, 0);
1103
1104         sunxi_nfc_hw_ecc_enable(nand);
1105
1106         for (i = 0; i < ecc->steps; i++) {
1107                 int data_off = i * ecc->size;
1108                 int oob_off = i * (ecc->bytes + 4);
1109                 u8 *data = buf + data_off;
1110                 u8 *oob = nand->oob_poi + oob_off;
1111
1112                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1113                                                   oob_off + mtd->writesize,
1114                                                   &cur_off, &max_bitflips,
1115                                                   !i, oob_required, page);
1116                 if (ret < 0)
1117                         return ret;
1118                 else if (ret)
1119                         raw_mode = true;
1120         }
1121
1122         if (oob_required)
1123                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1124                                                 !raw_mode, page);
1125
1126         sunxi_nfc_hw_ecc_disable(nand);
1127
1128         return max_bitflips;
1129 }
1130
1131 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1132                                           int oob_required, int page)
1133 {
1134         int ret;
1135
1136         sunxi_nfc_select_chip(nand, nand->cur_cs);
1137
1138         nand_read_page_op(nand, page, 0, NULL, 0);
1139
1140         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1141                                                nand->ecc.steps);
1142         if (ret >= 0)
1143                 return ret;
1144
1145         /* Fallback to PIO mode */
1146         return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1147 }
1148
1149 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1150                                          u32 data_offs, u32 readlen,
1151                                          u8 *bufpoi, int page)
1152 {
1153         struct mtd_info *mtd = nand_to_mtd(nand);
1154         struct nand_ecc_ctrl *ecc = &nand->ecc;
1155         int ret, i, cur_off = 0;
1156         unsigned int max_bitflips = 0;
1157
1158         sunxi_nfc_select_chip(nand, nand->cur_cs);
1159
1160         nand_read_page_op(nand, page, 0, NULL, 0);
1161
1162         sunxi_nfc_hw_ecc_enable(nand);
1163
1164         for (i = data_offs / ecc->size;
1165              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1166                 int data_off = i * ecc->size;
1167                 int oob_off = i * (ecc->bytes + 4);
1168                 u8 *data = bufpoi + data_off;
1169                 u8 *oob = nand->oob_poi + oob_off;
1170
1171                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1172                                                   oob,
1173                                                   oob_off + mtd->writesize,
1174                                                   &cur_off, &max_bitflips, !i,
1175                                                   false, page);
1176                 if (ret < 0)
1177                         return ret;
1178         }
1179
1180         sunxi_nfc_hw_ecc_disable(nand);
1181
1182         return max_bitflips;
1183 }
1184
1185 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1186                                              u32 data_offs, u32 readlen,
1187                                              u8 *buf, int page)
1188 {
1189         int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1190         int ret;
1191
1192         sunxi_nfc_select_chip(nand, nand->cur_cs);
1193
1194         nand_read_page_op(nand, page, 0, NULL, 0);
1195
1196         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1197         if (ret >= 0)
1198                 return ret;
1199
1200         /* Fallback to PIO mode */
1201         return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1202                                              buf, page);
1203 }
1204
1205 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1206                                        const uint8_t *buf, int oob_required,
1207                                        int page)
1208 {
1209         struct mtd_info *mtd = nand_to_mtd(nand);
1210         struct nand_ecc_ctrl *ecc = &nand->ecc;
1211         int ret, i, cur_off = 0;
1212
1213         sunxi_nfc_select_chip(nand, nand->cur_cs);
1214
1215         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1216
1217         sunxi_nfc_hw_ecc_enable(nand);
1218
1219         for (i = 0; i < ecc->steps; i++) {
1220                 int data_off = i * ecc->size;
1221                 int oob_off = i * (ecc->bytes + 4);
1222                 const u8 *data = buf + data_off;
1223                 const u8 *oob = nand->oob_poi + oob_off;
1224
1225                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1226                                                    oob_off + mtd->writesize,
1227                                                    &cur_off, !i, page);
1228                 if (ret)
1229                         return ret;
1230         }
1231
1232         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1233                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1234                                                  &cur_off, page);
1235
1236         sunxi_nfc_hw_ecc_disable(nand);
1237
1238         return nand_prog_page_end_op(nand);
1239 }
1240
1241 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1242                                           u32 data_offs, u32 data_len,
1243                                           const u8 *buf, int oob_required,
1244                                           int page)
1245 {
1246         struct mtd_info *mtd = nand_to_mtd(nand);
1247         struct nand_ecc_ctrl *ecc = &nand->ecc;
1248         int ret, i, cur_off = 0;
1249
1250         sunxi_nfc_select_chip(nand, nand->cur_cs);
1251
1252         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1253
1254         sunxi_nfc_hw_ecc_enable(nand);
1255
1256         for (i = data_offs / ecc->size;
1257              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1258                 int data_off = i * ecc->size;
1259                 int oob_off = i * (ecc->bytes + 4);
1260                 const u8 *data = buf + data_off;
1261                 const u8 *oob = nand->oob_poi + oob_off;
1262
1263                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1264                                                    oob_off + mtd->writesize,
1265                                                    &cur_off, !i, page);
1266                 if (ret)
1267                         return ret;
1268         }
1269
1270         sunxi_nfc_hw_ecc_disable(nand);
1271
1272         return nand_prog_page_end_op(nand);
1273 }
1274
1275 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1276                                            const u8 *buf,
1277                                            int oob_required,
1278                                            int page)
1279 {
1280         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1281         struct nand_ecc_ctrl *ecc = &nand->ecc;
1282         struct scatterlist sg;
1283         u32 wait;
1284         int ret, i;
1285
1286         sunxi_nfc_select_chip(nand, nand->cur_cs);
1287
1288         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1289         if (ret)
1290                 return ret;
1291
1292         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1293                                        DMA_TO_DEVICE, &sg);
1294         if (ret)
1295                 goto pio_fallback;
1296
1297         for (i = 0; i < ecc->steps; i++) {
1298                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1299
1300                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1301         }
1302
1303         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1304
1305         sunxi_nfc_hw_ecc_enable(nand);
1306         sunxi_nfc_randomizer_config(nand, page, false);
1307         sunxi_nfc_randomizer_enable(nand);
1308
1309         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1310                nfc->regs + NFC_REG_WCMD_SET);
1311
1312         wait = NFC_CMD_INT_FLAG;
1313
1314         if (nfc->caps->has_mdma)
1315                 wait |= NFC_DMA_INT_FLAG;
1316         else
1317                 dma_async_issue_pending(nfc->dmac);
1318
1319         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1320                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1321                nfc->regs + NFC_REG_CMD);
1322
1323         ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
1324         if (ret && !nfc->caps->has_mdma)
1325                 dmaengine_terminate_all(nfc->dmac);
1326
1327         sunxi_nfc_randomizer_disable(nand);
1328         sunxi_nfc_hw_ecc_disable(nand);
1329
1330         sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1331
1332         if (ret)
1333                 return ret;
1334
1335         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1336                 /* TODO: use DMA to transfer extra OOB bytes ? */
1337                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1338                                                  NULL, page);
1339
1340         return nand_prog_page_end_op(nand);
1341
1342 pio_fallback:
1343         return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1344 }
1345
1346 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1347 {
1348         u8 *buf = nand_get_data_buf(nand);
1349
1350         return nand->ecc.read_page(nand, buf, 1, page);
1351 }
1352
1353 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1354 {
1355         struct mtd_info *mtd = nand_to_mtd(nand);
1356         u8 *buf = nand_get_data_buf(nand);
1357         int ret;
1358
1359         memset(buf, 0xff, mtd->writesize);
1360         ret = nand->ecc.write_page(nand, buf, 1, page);
1361         if (ret)
1362                 return ret;
1363
1364         /* Send command to program the OOB data */
1365         return nand_prog_page_end_op(nand);
1366 }
1367
1368 static const s32 tWB_lut[] = {6, 12, 16, 20};
1369 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1370
1371 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1372                 u32 clk_period)
1373 {
1374         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1375         int i;
1376
1377         for (i = 0; i < lut_size; i++) {
1378                 if (clk_cycles <= lut[i])
1379                         return i;
1380         }
1381
1382         /* Doesn't fit */
1383         return -EINVAL;
1384 }
1385
1386 #define sunxi_nand_lookup_timing(l, p, c) \
1387                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1388
1389 static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
1390                                      const struct nand_interface_config *conf)
1391 {
1392         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1393         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1394         const struct nand_sdr_timings *timings;
1395         u32 min_clk_period = 0;
1396         s32 tWB, tADL, tWHR, tRHW, tCAD;
1397         long real_clk_rate;
1398
1399         timings = nand_get_sdr_timings(conf);
1400         if (IS_ERR(timings))
1401                 return -ENOTSUPP;
1402
1403         /* T1 <=> tCLS */
1404         if (timings->tCLS_min > min_clk_period)
1405                 min_clk_period = timings->tCLS_min;
1406
1407         /* T2 <=> tCLH */
1408         if (timings->tCLH_min > min_clk_period)
1409                 min_clk_period = timings->tCLH_min;
1410
1411         /* T3 <=> tCS */
1412         if (timings->tCS_min > min_clk_period)
1413                 min_clk_period = timings->tCS_min;
1414
1415         /* T4 <=> tCH */
1416         if (timings->tCH_min > min_clk_period)
1417                 min_clk_period = timings->tCH_min;
1418
1419         /* T5 <=> tWP */
1420         if (timings->tWP_min > min_clk_period)
1421                 min_clk_period = timings->tWP_min;
1422
1423         /* T6 <=> tWH */
1424         if (timings->tWH_min > min_clk_period)
1425                 min_clk_period = timings->tWH_min;
1426
1427         /* T7 <=> tALS */
1428         if (timings->tALS_min > min_clk_period)
1429                 min_clk_period = timings->tALS_min;
1430
1431         /* T8 <=> tDS */
1432         if (timings->tDS_min > min_clk_period)
1433                 min_clk_period = timings->tDS_min;
1434
1435         /* T9 <=> tDH */
1436         if (timings->tDH_min > min_clk_period)
1437                 min_clk_period = timings->tDH_min;
1438
1439         /* T10 <=> tRR */
1440         if (timings->tRR_min > (min_clk_period * 3))
1441                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1442
1443         /* T11 <=> tALH */
1444         if (timings->tALH_min > min_clk_period)
1445                 min_clk_period = timings->tALH_min;
1446
1447         /* T12 <=> tRP */
1448         if (timings->tRP_min > min_clk_period)
1449                 min_clk_period = timings->tRP_min;
1450
1451         /* T13 <=> tREH */
1452         if (timings->tREH_min > min_clk_period)
1453                 min_clk_period = timings->tREH_min;
1454
1455         /* T14 <=> tRC */
1456         if (timings->tRC_min > (min_clk_period * 2))
1457                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1458
1459         /* T15 <=> tWC */
1460         if (timings->tWC_min > (min_clk_period * 2))
1461                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1462
1463         /* T16 - T19 + tCAD */
1464         if (timings->tWB_max > (min_clk_period * 20))
1465                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1466
1467         if (timings->tADL_min > (min_clk_period * 32))
1468                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1469
1470         if (timings->tWHR_min > (min_clk_period * 32))
1471                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1472
1473         if (timings->tRHW_min > (min_clk_period * 20))
1474                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1475
1476         /*
1477          * In non-EDO, tREA should be less than tRP to guarantee that the
1478          * controller does not sample the IO lines too early. Unfortunately,
1479          * the sunxi NAND controller does not allow us to have different
1480          * values for tRP and tREH (tRP = tREH = tRW / 2).
1481          *
1482          * We have 2 options to overcome this limitation:
1483          *
1484          * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1485          * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1486          */
1487         if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1488                 min_clk_period = timings->tREA_max;
1489
1490         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1491                                         min_clk_period);
1492         if (tWB < 0) {
1493                 dev_err(nfc->dev, "unsupported tWB\n");
1494                 return tWB;
1495         }
1496
1497         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1498         if (tADL > 3) {
1499                 dev_err(nfc->dev, "unsupported tADL\n");
1500                 return -EINVAL;
1501         }
1502
1503         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1504         if (tWHR > 3) {
1505                 dev_err(nfc->dev, "unsupported tWHR\n");
1506                 return -EINVAL;
1507         }
1508
1509         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1510                                         min_clk_period);
1511         if (tRHW < 0) {
1512                 dev_err(nfc->dev, "unsupported tRHW\n");
1513                 return tRHW;
1514         }
1515
1516         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1517                 return 0;
1518
1519         /*
1520          * TODO: according to ONFI specs this value only applies for DDR NAND,
1521          * but Allwinner seems to set this to 0x7. Mimic them for now.
1522          */
1523         tCAD = 0x7;
1524
1525         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1526         sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1527
1528         /* Convert min_clk_period from picoseconds to nanoseconds */
1529         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1530
1531         /*
1532          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1533          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1534          * This new formula was verified with a scope and validated by
1535          * Allwinner engineers.
1536          */
1537         sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1538         real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1539         if (real_clk_rate <= 0) {
1540                 dev_err(nfc->dev, "Unable to round clk %lu\n",
1541                         sunxi_nand->clk_rate);
1542                 return -EINVAL;
1543         }
1544
1545         sunxi_nand->timing_ctl = 0;
1546
1547         /*
1548          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1549          * output cycle timings shall be used if the host drives tRC less than
1550          * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1551          */
1552         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1553         if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1554                 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1555
1556         return 0;
1557 }
1558
1559 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1560                                     struct mtd_oob_region *oobregion)
1561 {
1562         struct nand_chip *nand = mtd_to_nand(mtd);
1563         struct nand_ecc_ctrl *ecc = &nand->ecc;
1564
1565         if (section >= ecc->steps)
1566                 return -ERANGE;
1567
1568         oobregion->offset = section * (ecc->bytes + 4) + 4;
1569         oobregion->length = ecc->bytes;
1570
1571         return 0;
1572 }
1573
1574 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1575                                      struct mtd_oob_region *oobregion)
1576 {
1577         struct nand_chip *nand = mtd_to_nand(mtd);
1578         struct nand_ecc_ctrl *ecc = &nand->ecc;
1579
1580         if (section > ecc->steps)
1581                 return -ERANGE;
1582
1583         /*
1584          * The first 2 bytes are used for BB markers, hence we
1585          * only have 2 bytes available in the first user data
1586          * section.
1587          */
1588         if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
1589                 oobregion->offset = 2;
1590                 oobregion->length = 2;
1591
1592                 return 0;
1593         }
1594
1595         /*
1596          * The controller does not provide access to OOB bytes
1597          * past the end of the ECC data.
1598          */
1599         if (section == ecc->steps && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST)
1600                 return -ERANGE;
1601
1602         oobregion->offset = section * (ecc->bytes + 4);
1603
1604         if (section < ecc->steps)
1605                 oobregion->length = 4;
1606         else
1607                 oobregion->length = mtd->oobsize - oobregion->offset;
1608
1609         return 0;
1610 }
1611
1612 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1613         .ecc = sunxi_nand_ooblayout_ecc,
1614         .free = sunxi_nand_ooblayout_free,
1615 };
1616
1617 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1618                                        struct nand_ecc_ctrl *ecc,
1619                                        struct device_node *np)
1620 {
1621         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1622         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1623         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1624         struct mtd_info *mtd = nand_to_mtd(nand);
1625         struct nand_device *nanddev = mtd_to_nanddev(mtd);
1626         int nsectors;
1627         int i;
1628
1629         if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
1630                 int bytes;
1631
1632                 ecc->size = 1024;
1633                 nsectors = mtd->writesize / ecc->size;
1634
1635                 /* Reserve 2 bytes for the BBM */
1636                 bytes = (mtd->oobsize - 2) / nsectors;
1637
1638                 /* 4 non-ECC bytes are added before each ECC bytes section */
1639                 bytes -= 4;
1640
1641                 /* and bytes has to be even. */
1642                 if (bytes % 2)
1643                         bytes--;
1644
1645                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1646
1647                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1648                         if (strengths[i] > ecc->strength)
1649                                 break;
1650                 }
1651
1652                 if (!i)
1653                         ecc->strength = 0;
1654                 else
1655                         ecc->strength = strengths[i - 1];
1656         }
1657
1658         if (ecc->size != 512 && ecc->size != 1024)
1659                 return -EINVAL;
1660
1661         /* Prefer 1k ECC chunk over 512 ones */
1662         if (ecc->size == 512 && mtd->writesize > 512) {
1663                 ecc->size = 1024;
1664                 ecc->strength *= 2;
1665         }
1666
1667         /* Add ECC info retrieval from DT */
1668         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1669                 if (ecc->strength <= strengths[i]) {
1670                         /*
1671                          * Update ecc->strength value with the actual strength
1672                          * that will be used by the ECC engine.
1673                          */
1674                         ecc->strength = strengths[i];
1675                         break;
1676                 }
1677         }
1678
1679         if (i >= ARRAY_SIZE(strengths)) {
1680                 dev_err(nfc->dev, "unsupported strength\n");
1681                 return -ENOTSUPP;
1682         }
1683
1684         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1685         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1686
1687         /* HW ECC always work with even numbers of ECC bytes */
1688         ecc->bytes = ALIGN(ecc->bytes, 2);
1689
1690         nsectors = mtd->writesize / ecc->size;
1691
1692         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors))
1693                 return -EINVAL;
1694
1695         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1696         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1697         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1698
1699         if (nfc->dmac || nfc->caps->has_mdma) {
1700                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1701                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1702                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1703                 nand->options |= NAND_USES_DMA;
1704         } else {
1705                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1706                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1707                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1708         }
1709
1710         /* TODO: support DMA for raw accesses and subpage write */
1711         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1712         ecc->read_oob_raw = nand_read_oob_std;
1713         ecc->write_oob_raw = nand_write_oob_std;
1714
1715         sunxi_nand->ecc.ecc_ctl = NFC_ECC_MODE(i) | NFC_ECC_EXCEPTION |
1716                                   NFC_ECC_PIPELINE | NFC_ECC_EN;
1717
1718         if (ecc->size == 512)
1719                 sunxi_nand->ecc.ecc_ctl |= NFC_ECC_BLOCK_512;
1720
1721         return 0;
1722 }
1723
1724 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1725 {
1726         const struct nand_ecc_props *requirements =
1727                 nanddev_get_ecc_requirements(&nand->base);
1728         struct nand_ecc_ctrl *ecc = &nand->ecc;
1729         struct device_node *np = nand_get_flash_node(nand);
1730         int ret;
1731
1732         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1733                 nand->bbt_options |= NAND_BBT_NO_OOB;
1734
1735         if (nand->options & NAND_NEED_SCRAMBLING)
1736                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1737
1738         nand->options |= NAND_SUBPAGE_READ;
1739
1740         if (!ecc->size) {
1741                 ecc->size = requirements->step_size;
1742                 ecc->strength = requirements->strength;
1743         }
1744
1745         if (!ecc->size || !ecc->strength)
1746                 return -EINVAL;
1747
1748         switch (ecc->engine_type) {
1749         case NAND_ECC_ENGINE_TYPE_ON_HOST:
1750                 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1751                 if (ret)
1752                         return ret;
1753                 break;
1754         case NAND_ECC_ENGINE_TYPE_NONE:
1755         case NAND_ECC_ENGINE_TYPE_SOFT:
1756                 break;
1757         default:
1758                 return -EINVAL;
1759         }
1760
1761         return 0;
1762 }
1763
1764 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1765                                 const struct nand_subop *subop)
1766 {
1767         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1768         u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1769         unsigned int i, j, remaining, start;
1770         void *inbuf = NULL;
1771         int ret;
1772
1773         for (i = 0; i < subop->ninstrs; i++) {
1774                 const struct nand_op_instr *instr = &subop->instrs[i];
1775
1776                 switch (instr->type) {
1777                 case NAND_OP_CMD_INSTR:
1778                         if (cmd & NFC_SEND_CMD1) {
1779                                 if (WARN_ON(cmd & NFC_SEND_CMD2))
1780                                         return -EINVAL;
1781
1782                                 cmd |= NFC_SEND_CMD2;
1783                                 extcmd |= instr->ctx.cmd.opcode;
1784                         } else {
1785                                 cmd |= NFC_SEND_CMD1 |
1786                                        NFC_CMD(instr->ctx.cmd.opcode);
1787                         }
1788                         break;
1789
1790                 case NAND_OP_ADDR_INSTR:
1791                         remaining = nand_subop_get_num_addr_cyc(subop, i);
1792                         start = nand_subop_get_addr_start_off(subop, i);
1793                         for (j = 0; j < 8 && j + start < remaining; j++) {
1794                                 u32 addr = instr->ctx.addr.addrs[j + start];
1795
1796                                 addrs[j / 4] |= addr << (j % 4) * 8;
1797                         }
1798
1799                         if (j)
1800                                 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1801
1802                         break;
1803
1804                 case NAND_OP_DATA_IN_INSTR:
1805                 case NAND_OP_DATA_OUT_INSTR:
1806                         start = nand_subop_get_data_start_off(subop, i);
1807                         remaining = nand_subop_get_data_len(subop, i);
1808                         cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1809                         cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1810
1811                         if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1812                                 cmd |= NFC_ACCESS_DIR;
1813                                 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1814                                             instr->ctx.data.buf.out + start,
1815                                             cnt);
1816                         } else {
1817                                 inbuf = instr->ctx.data.buf.in + start;
1818                         }
1819
1820                         break;
1821
1822                 case NAND_OP_WAITRDY_INSTR:
1823                         cmd |= NFC_WAIT_FLAG;
1824                         break;
1825                 }
1826         }
1827
1828         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1829         if (ret)
1830                 return ret;
1831
1832         if (cmd & NFC_SEND_ADR) {
1833                 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1834                 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1835         }
1836
1837         if (cmd & NFC_SEND_CMD2)
1838                 writel(extcmd,
1839                        nfc->regs +
1840                        (cmd & NFC_ACCESS_DIR ?
1841                         NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1842
1843         if (cmd & NFC_DATA_TRANS)
1844                 writel(cnt, nfc->regs + NFC_REG_CNT);
1845
1846         writel(cmd, nfc->regs + NFC_REG_CMD);
1847
1848         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1849                                     !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1850                                     0);
1851         if (ret)
1852                 return ret;
1853
1854         if (inbuf)
1855                 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1856
1857         return 0;
1858 }
1859
1860 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1861                                   const struct nand_subop *subop)
1862 {
1863         return nand_soft_waitrdy(nand,
1864                                  subop->instrs[0].ctx.waitrdy.timeout_ms);
1865 }
1866
1867 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1868         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1869                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1871                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1872                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1873                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1874         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1875                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1876                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1877                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1878                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1879                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1880 );
1881
1882 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1883         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1884                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1885                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1886                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1887                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1888         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1889                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1890                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1891                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1892                                NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1893         NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1894                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1895 );
1896
1897 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1898                              const struct nand_operation *op, bool check_only)
1899 {
1900         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1901         const struct nand_op_parser *parser;
1902
1903         if (!check_only)
1904                 sunxi_nfc_select_chip(nand, op->cs);
1905
1906         if (sunxi_nand->sels[op->cs].rb >= 0)
1907                 parser = &sunxi_nfc_op_parser;
1908         else
1909                 parser = &sunxi_nfc_norb_op_parser;
1910
1911         return nand_op_parser_exec_op(nand, parser, op, check_only);
1912 }
1913
1914 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1915         .attach_chip = sunxi_nand_attach_chip,
1916         .setup_interface = sunxi_nfc_setup_interface,
1917         .exec_op = sunxi_nfc_exec_op,
1918 };
1919
1920 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1921 {
1922         struct sunxi_nand_chip *sunxi_nand;
1923         struct nand_chip *chip;
1924         int ret;
1925
1926         while (!list_empty(&nfc->chips)) {
1927                 sunxi_nand = list_first_entry(&nfc->chips,
1928                                               struct sunxi_nand_chip,
1929                                               node);
1930                 chip = &sunxi_nand->nand;
1931                 ret = mtd_device_unregister(nand_to_mtd(chip));
1932                 WARN_ON(ret);
1933                 nand_cleanup(chip);
1934                 list_del(&sunxi_nand->node);
1935         }
1936 }
1937
1938 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1939                                 struct device_node *np)
1940 {
1941         struct sunxi_nand_chip *sunxi_nand;
1942         struct mtd_info *mtd;
1943         struct nand_chip *nand;
1944         int nsels;
1945         int ret;
1946         int i;
1947         u32 tmp;
1948
1949         if (!of_get_property(np, "reg", &nsels))
1950                 return -EINVAL;
1951
1952         nsels /= sizeof(u32);
1953         if (!nsels) {
1954                 dev_err(dev, "invalid reg property size\n");
1955                 return -EINVAL;
1956         }
1957
1958         sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1959                                   GFP_KERNEL);
1960         if (!sunxi_nand)
1961                 return -ENOMEM;
1962
1963         sunxi_nand->nsels = nsels;
1964
1965         for (i = 0; i < nsels; i++) {
1966                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1967                 if (ret) {
1968                         dev_err(dev, "could not retrieve reg property: %d\n",
1969                                 ret);
1970                         return ret;
1971                 }
1972
1973                 if (tmp > NFC_MAX_CS) {
1974                         dev_err(dev,
1975                                 "invalid reg value: %u (max CS = 7)\n",
1976                                 tmp);
1977                         return -EINVAL;
1978                 }
1979
1980                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1981                         dev_err(dev, "CS %d already assigned\n", tmp);
1982                         return -EINVAL;
1983                 }
1984
1985                 sunxi_nand->sels[i].cs = tmp;
1986
1987                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1988                     tmp < 2)
1989                         sunxi_nand->sels[i].rb = tmp;
1990                 else
1991                         sunxi_nand->sels[i].rb = -1;
1992         }
1993
1994         nand = &sunxi_nand->nand;
1995         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1996         nand->controller = &nfc->controller;
1997         nand->controller->ops = &sunxi_nand_controller_ops;
1998
1999         /*
2000          * Set the ECC mode to the default value in case nothing is specified
2001          * in the DT.
2002          */
2003         nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2004         nand_set_flash_node(nand, np);
2005
2006         mtd = nand_to_mtd(nand);
2007         mtd->dev.parent = dev;
2008
2009         ret = nand_scan(nand, nsels);
2010         if (ret)
2011                 return ret;
2012
2013         ret = mtd_device_register(mtd, NULL, 0);
2014         if (ret) {
2015                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2016                 nand_cleanup(nand);
2017                 return ret;
2018         }
2019
2020         list_add_tail(&sunxi_nand->node, &nfc->chips);
2021
2022         return 0;
2023 }
2024
2025 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2026 {
2027         struct device_node *np = dev->of_node;
2028         int ret;
2029
2030         for_each_child_of_node_scoped(np, nand_np) {
2031                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2032                 if (ret) {
2033                         sunxi_nand_chips_cleanup(nfc);
2034                         return ret;
2035                 }
2036         }
2037
2038         return 0;
2039 }
2040
2041 static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)
2042 {
2043         int ret;
2044
2045         if (nfc->caps->has_mdma)
2046                 return 0;
2047
2048         nfc->dmac = dma_request_chan(nfc->dev, "rxtx");
2049         if (IS_ERR(nfc->dmac)) {
2050                 ret = PTR_ERR(nfc->dmac);
2051                 if (ret == -EPROBE_DEFER)
2052                         return ret;
2053
2054                 /* Ignore errors to fall back to PIO mode */
2055                 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);
2056                 nfc->dmac = NULL;
2057         } else {
2058                 struct dma_slave_config dmac_cfg = { };
2059
2060                 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2061                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2062                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2063                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2064                 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2065                 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2066                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2067         }
2068         return 0;
2069 }
2070
2071 static int sunxi_nfc_probe(struct platform_device *pdev)
2072 {
2073         struct device *dev = &pdev->dev;
2074         struct resource *r;
2075         struct sunxi_nfc *nfc;
2076         int irq;
2077         int ret;
2078
2079         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2080         if (!nfc)
2081                 return -ENOMEM;
2082
2083         nfc->dev = dev;
2084         nand_controller_init(&nfc->controller);
2085         INIT_LIST_HEAD(&nfc->chips);
2086
2087         nfc->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
2088         if (IS_ERR(nfc->regs))
2089                 return PTR_ERR(nfc->regs);
2090
2091         irq = platform_get_irq(pdev, 0);
2092         if (irq < 0)
2093                 return irq;
2094
2095         nfc->ahb_clk = devm_clk_get_enabled(dev, "ahb");
2096         if (IS_ERR(nfc->ahb_clk)) {
2097                 dev_err(dev, "failed to retrieve ahb clk\n");
2098                 return PTR_ERR(nfc->ahb_clk);
2099         }
2100
2101         nfc->mod_clk = devm_clk_get_enabled(dev, "mod");
2102         if (IS_ERR(nfc->mod_clk)) {
2103                 dev_err(dev, "failed to retrieve mod clk\n");
2104                 return PTR_ERR(nfc->mod_clk);
2105         }
2106
2107         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2108         if (IS_ERR(nfc->reset))
2109                 return PTR_ERR(nfc->reset);
2110
2111         ret = reset_control_deassert(nfc->reset);
2112         if (ret) {
2113                 dev_err(dev, "reset err %d\n", ret);
2114                 return ret;
2115         }
2116
2117         nfc->caps = of_device_get_match_data(&pdev->dev);
2118         if (!nfc->caps) {
2119                 ret = -EINVAL;
2120                 goto out_ahb_reset_reassert;
2121         }
2122
2123         ret = sunxi_nfc_rst(nfc);
2124         if (ret)
2125                 goto out_ahb_reset_reassert;
2126
2127         writel(0, nfc->regs + NFC_REG_INT);
2128         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2129                                0, "sunxi-nand", nfc);
2130         if (ret)
2131                 goto out_ahb_reset_reassert;
2132
2133         ret = sunxi_nfc_dma_init(nfc, r);
2134
2135         if (ret)
2136                 goto out_ahb_reset_reassert;
2137
2138         platform_set_drvdata(pdev, nfc);
2139
2140         ret = sunxi_nand_chips_init(dev, nfc);
2141         if (ret) {
2142                 dev_err(dev, "failed to init nand chips\n");
2143                 goto out_release_dmac;
2144         }
2145
2146         return 0;
2147
2148 out_release_dmac:
2149         if (nfc->dmac)
2150                 dma_release_channel(nfc->dmac);
2151 out_ahb_reset_reassert:
2152         reset_control_assert(nfc->reset);
2153
2154         return ret;
2155 }
2156
2157 static void sunxi_nfc_remove(struct platform_device *pdev)
2158 {
2159         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2160
2161         sunxi_nand_chips_cleanup(nfc);
2162
2163         reset_control_assert(nfc->reset);
2164
2165         if (nfc->dmac)
2166                 dma_release_channel(nfc->dmac);
2167 }
2168
2169 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2170         .reg_io_data = NFC_REG_A10_IO_DATA,
2171         .dma_maxburst = 4,
2172 };
2173
2174 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
2175         .has_mdma = true,
2176         .reg_io_data = NFC_REG_A23_IO_DATA,
2177         .dma_maxburst = 8,
2178 };
2179
2180 static const struct of_device_id sunxi_nfc_ids[] = {
2181         {
2182                 .compatible = "allwinner,sun4i-a10-nand",
2183                 .data = &sunxi_nfc_a10_caps,
2184         },
2185         {
2186                 .compatible = "allwinner,sun8i-a23-nand-controller",
2187                 .data = &sunxi_nfc_a23_caps,
2188         },
2189         { /* sentinel */ }
2190 };
2191 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2192
2193 static struct platform_driver sunxi_nfc_driver = {
2194         .driver = {
2195                 .name = "sunxi_nand",
2196                 .of_match_table = sunxi_nfc_ids,
2197         },
2198         .probe = sunxi_nfc_probe,
2199         .remove = sunxi_nfc_remove,
2200 };
2201 module_platform_driver(sunxi_nfc_driver);
2202
2203 MODULE_LICENSE("GPL");
2204 MODULE_AUTHOR("Boris BREZILLON");
2205 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2206 MODULE_ALIAS("platform:sunxi_nand");