target/linux/mediatek/patches-5.15/120-12-v5.19-spi-add-driver-for-MTK-SPI-NAND-Flash-Interface.patch

   1 From 8170bafa8936e9fbfdce992932a63bd20eca3bc3 Mon Sep 17 00:00:00 2001
   2 From: Chuanhong Guo <gch981213@gmail.com>
   3 Date: Sat, 2 Apr 2022 10:16:11 +0800
   4 Subject: [PATCH v6 2/5] spi: add driver for MTK SPI NAND Flash Interface
   5
   6 This driver implements support for the SPI-NAND mode of MTK NAND Flash
   7 Interface as a SPI-MEM controller with pipelined ECC capability.
   8
   9 Signed-off-by: Chuanhong Guo <gch981213@gmail.com>
  10 Tested-by: Daniel Golle <daniel@makrotopia.org>
  11 ---
  12 Change since v1:
  13   fix CI warnings
  14
  15 Changes since v2:
  16  use streamed DMA api to avoid an extra memory copy during read
  17  make ECC engine config a per-nand context
  18  take user-requested ECC strength into account
  19
  20 Change since v3: none
  21 Changes since v4:
  22  fix missing OOB write
  23  print page format with dev_dbg
  24  replace uint*_t copied from vendor driver with u*
  25
  26 Changes since v5:
  27  add missing nfi mode register configuration in probe
  28  fix an off-by-one bug in mtk_snand_mac_io
  29
  30  drivers/spi/Kconfig        |   10 +
  31  drivers/spi/Makefile       |    1 +
  32  drivers/spi/spi-mtk-snfi.c | 1470 ++++++++++++++++++++++++++++++++++++
  33  3 files changed, 1481 insertions(+)
  34  create mode 100644 drivers/spi/spi-mtk-snfi.c
  35
  36 --- a/drivers/spi/Kconfig
  37 +++ b/drivers/spi/Kconfig
  38 @@ -529,6 +529,16 @@ config SPI_MTK_NOR
  39           SPI interface as well as several SPI NOR specific instructions
  40           via SPI MEM interface.
  41
  42 +config SPI_MTK_SNFI
  43 +       tristate "MediaTek SPI NAND Flash Interface"
  44 +       depends on ARCH_MEDIATEK || COMPILE_TEST
  45 +       depends on MTD_NAND_ECC_MEDIATEK
  46 +       help
  47 +         This enables support for SPI-NAND mode on the MediaTek NAND
  48 +         Flash Interface found on MediaTek ARM SoCs. This controller
  49 +         is implemented as a SPI-MEM controller with pipelined ECC
  50 +         capcability.
  51 +
  52  config SPI_NPCM_FIU
  53         tristate "Nuvoton NPCM FLASH Interface Unit"
  54         depends on ARCH_NPCM || COMPILE_TEST
  55 --- a/drivers/spi/Makefile
  56 +++ b/drivers/spi/Makefile
  57 @@ -71,6 +71,7 @@ obj-$(CONFIG_SPI_MPC52xx)             += spi-mpc52x
  58  obj-$(CONFIG_SPI_MT65XX)                += spi-mt65xx.o
  59  obj-$(CONFIG_SPI_MT7621)               += spi-mt7621.o
  60  obj-$(CONFIG_SPI_MTK_NOR)              += spi-mtk-nor.o
  61 +obj-$(CONFIG_SPI_MTK_SNFI)             += spi-mtk-snfi.o
  62  obj-$(CONFIG_SPI_MXIC)                 += spi-mxic.o
  63  obj-$(CONFIG_SPI_MXS)                  += spi-mxs.o
  64  obj-$(CONFIG_SPI_NPCM_FIU)             += spi-npcm-fiu.o
  65 --- /dev/null
  66 +++ b/drivers/spi/spi-mtk-snfi.c
  67 @@ -0,0 +1,1470 @@
  68 +// SPDX-License-Identifier: GPL-2.0
  69 +//
  70 +// Driver for the SPI-NAND mode of Mediatek NAND Flash Interface
  71 +//
  72 +// Copyright (c) 2022 Chuanhong Guo <gch981213@gmail.com>
  73 +//
  74 +// This driver is based on the SPI-NAND mtd driver from Mediatek SDK:
  75 +//
  76 +// Copyright (C) 2020 MediaTek Inc.
  77 +// Author: Weijie Gao <weijie.gao@mediatek.com>
  78 +//
  79 +// This controller organize the page data as several interleaved sectors
  80 +// like the following: (sizeof(FDM + ECC) = snf->nfi_cfg.spare_size)
  81 +// +---------+------+------+---------+------+------+-----+
  82 +// | Sector1 | FDM1 | ECC1 | Sector2 | FDM2 | ECC2 | ... |
  83 +// +---------+------+------+---------+------+------+-----+
  84 +// With auto-format turned on, DMA only returns this part:
  85 +// +---------+---------+-----+
  86 +// | Sector1 | Sector2 | ... |
  87 +// +---------+---------+-----+
  88 +// The FDM data will be filled to the registers, and ECC parity data isn't
  89 +// accessible.
  90 +// With auto-format off, all ((Sector+FDM+ECC)*nsectors) will be read over DMA
  91 +// in it's original order shown in the first table. ECC can't be turned on when
  92 +// auto-format is off.
  93 +//
  94 +// However, Linux SPI-NAND driver expects the data returned as:
  95 +// +------+-----+
  96 +// | Page | OOB |
  97 +// +------+-----+
  98 +// where the page data is continuously stored instead of interleaved.
  99 +// So we assume all instructions matching the page_op template between ECC
 100 +// prepare_io_req and finish_io_req are for page cache r/w.
 101 +// Here's how this spi-mem driver operates when reading:
 102 +//  1. Always set snf->autofmt = true in prepare_io_req (even when ECC is off).
 103 +//  2. Perform page ops and let the controller fill the DMA bounce buffer with
 104 +//     de-interleaved sector data and set FDM registers.
 105 +//  3. Return the data as:
 106 +//     +---------+---------+-----+------+------+-----+
 107 +//     | Sector1 | Sector2 | ... | FDM1 | FDM2 | ... |
 108 +//     +---------+---------+-----+------+------+-----+
 109 +//  4. For other matching spi_mem ops outside a prepare/finish_io_req pair,
 110 +//     read the data with auto-format off into the bounce buffer and copy
 111 +//     needed data to the buffer specified in the request.
 112 +//
 113 +// Write requests operates in a similar manner.
 114 +// As a limitation of this strategy, we won't be able to access any ECC parity
 115 +// data at all in Linux.
 116 +//
 117 +// Here's the bad block mark situation on MTK chips:
 118 +// In older chips like mt7622, MTK uses the first FDM byte in the first sector
 119 +// as the bad block mark. After de-interleaving, this byte appears at [pagesize]
 120 +// in the returned data, which is the BBM position expected by kernel. However,
 121 +// the conventional bad block mark is the first byte of the OOB, which is part
 122 +// of the last sector data in the interleaved layout. Instead of fixing their
 123 +// hardware, MTK decided to address this inconsistency in software. On these
 124 +// later chips, the BootROM expects the following:
 125 +// 1. The [pagesize] byte on a nand page is used as BBM, which will appear at
 126 +//    (page_size - (nsectors - 1) * spare_size) in the DMA buffer.
 127 +// 2. The original byte stored at that position in the DMA buffer will be stored
 128 +//    as the first byte of the FDM section in the last sector.
 129 +// We can't disagree with the BootROM, so after de-interleaving, we need to
 130 +// perform the following swaps in read:
 131 +// 1. Store the BBM at [page_size - (nsectors - 1) * spare_size] to [page_size],
 132 +//    which is the expected BBM position by kernel.
 133 +// 2. Store the page data byte at [pagesize + (nsectors-1) * fdm] back to
 134 +//    [page_size - (nsectors - 1) * spare_size]
 135 +// Similarly, when writing, we need to perform swaps in the other direction.
 136 +
 137 +#include <linux/kernel.h>
 138 +#include <linux/module.h>
 139 +#include <linux/init.h>
 140 +#include <linux/device.h>
 141 +#include <linux/mutex.h>
 142 +#include <linux/clk.h>
 143 +#include <linux/interrupt.h>
 144 +#include <linux/dma-mapping.h>
 145 +#include <linux/iopoll.h>
 146 +#include <linux/of_platform.h>
 147 +#include <linux/mtd/nand-ecc-mtk.h>
 148 +#include <linux/spi/spi.h>
 149 +#include <linux/spi/spi-mem.h>
 150 +#include <linux/mtd/nand.h>
 151 +
 152 +// NFI registers
 153 +#define NFI_CNFG 0x000
 154 +#define CNFG_OP_MODE_S 12
 155 +#define CNFG_OP_MODE_CUST 6
 156 +#define CNFG_OP_MODE_PROGRAM 3
 157 +#define CNFG_AUTO_FMT_EN BIT(9)
 158 +#define CNFG_HW_ECC_EN BIT(8)
 159 +#define CNFG_DMA_BURST_EN BIT(2)
 160 +#define CNFG_READ_MODE BIT(1)
 161 +#define CNFG_DMA_MODE BIT(0)
 162 +
 163 +#define NFI_PAGEFMT 0x0004
 164 +#define NFI_SPARE_SIZE_LS_S 16
 165 +#define NFI_FDM_ECC_NUM_S 12
 166 +#define NFI_FDM_NUM_S 8
 167 +#define NFI_SPARE_SIZE_S 4
 168 +#define NFI_SEC_SEL_512 BIT(2)
 169 +#define NFI_PAGE_SIZE_S 0
 170 +#define NFI_PAGE_SIZE_512_2K 0
 171 +#define NFI_PAGE_SIZE_2K_4K 1
 172 +#define NFI_PAGE_SIZE_4K_8K 2
 173 +#define NFI_PAGE_SIZE_8K_16K 3
 174 +
 175 +#define NFI_CON 0x008
 176 +#define CON_SEC_NUM_S 12
 177 +#define CON_BWR BIT(9)
 178 +#define CON_BRD BIT(8)
 179 +#define CON_NFI_RST BIT(1)
 180 +#define CON_FIFO_FLUSH BIT(0)
 181 +
 182 +#define NFI_INTR_EN 0x010
 183 +#define NFI_INTR_STA 0x014
 184 +#define NFI_IRQ_INTR_EN BIT(31)
 185 +#define NFI_IRQ_CUS_READ BIT(8)
 186 +#define NFI_IRQ_CUS_PG BIT(7)
 187 +
 188 +#define NFI_CMD 0x020
 189 +#define NFI_CMD_DUMMY_READ 0x00
 190 +#define NFI_CMD_DUMMY_WRITE 0x80
 191 +
 192 +#define NFI_STRDATA 0x040
 193 +#define STR_DATA BIT(0)
 194 +
 195 +#define NFI_STA 0x060
 196 +#define NFI_NAND_FSM GENMASK(28, 24)
 197 +#define NFI_FSM GENMASK(19, 16)
 198 +#define READ_EMPTY BIT(12)
 199 +
 200 +#define NFI_FIFOSTA 0x064
 201 +#define FIFO_WR_REMAIN_S 8
 202 +#define FIFO_RD_REMAIN_S 0
 203 +
 204 +#define NFI_ADDRCNTR 0x070
 205 +#define SEC_CNTR GENMASK(16, 12)
 206 +#define SEC_CNTR_S 12
 207 +#define NFI_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
 208 +
 209 +#define NFI_STRADDR 0x080
 210 +
 211 +#define NFI_BYTELEN 0x084
 212 +#define BUS_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
 213 +
 214 +#define NFI_FDM0L 0x0a0
 215 +#define NFI_FDM0M 0x0a4
 216 +#define NFI_FDML(n) (NFI_FDM0L + (n)*8)
 217 +#define NFI_FDMM(n) (NFI_FDM0M + (n)*8)
 218 +
 219 +#define NFI_DEBUG_CON1 0x220
 220 +#define WBUF_EN BIT(2)
 221 +
 222 +#define NFI_MASTERSTA 0x224
 223 +#define MAS_ADDR GENMASK(11, 9)
 224 +#define MAS_RD GENMASK(8, 6)
 225 +#define MAS_WR GENMASK(5, 3)
 226 +#define MAS_RDDLY GENMASK(2, 0)
 227 +#define NFI_MASTERSTA_MASK_7622 (MAS_ADDR | MAS_RD | MAS_WR | MAS_RDDLY)
 228 +
 229 +// SNFI registers
 230 +#define SNF_MAC_CTL 0x500
 231 +#define MAC_XIO_SEL BIT(4)
 232 +#define SF_MAC_EN BIT(3)
 233 +#define SF_TRIG BIT(2)
 234 +#define WIP_READY BIT(1)
 235 +#define WIP BIT(0)
 236 +
 237 +#define SNF_MAC_OUTL 0x504
 238 +#define SNF_MAC_INL 0x508
 239 +
 240 +#define SNF_RD_CTL2 0x510
 241 +#define DATA_READ_DUMMY_S 8
 242 +#define DATA_READ_MAX_DUMMY 0xf
 243 +#define DATA_READ_CMD_S 0
 244 +
 245 +#define SNF_RD_CTL3 0x514
 246 +
 247 +#define SNF_PG_CTL1 0x524
 248 +#define PG_LOAD_CMD_S 8
 249 +
 250 +#define SNF_PG_CTL2 0x528
 251 +
 252 +#define SNF_MISC_CTL 0x538
 253 +#define SW_RST BIT(28)
 254 +#define FIFO_RD_LTC_S 25
 255 +#define PG_LOAD_X4_EN BIT(20)
 256 +#define DATA_READ_MODE_S 16
 257 +#define DATA_READ_MODE GENMASK(18, 16)
 258 +#define DATA_READ_MODE_X1 0
 259 +#define DATA_READ_MODE_X2 1
 260 +#define DATA_READ_MODE_X4 2
 261 +#define DATA_READ_MODE_DUAL 5
 262 +#define DATA_READ_MODE_QUAD 6
 263 +#define PG_LOAD_CUSTOM_EN BIT(7)
 264 +#define DATARD_CUSTOM_EN BIT(6)
 265 +#define CS_DESELECT_CYC_S 0
 266 +
 267 +#define SNF_MISC_CTL2 0x53c
 268 +#define PROGRAM_LOAD_BYTE_NUM_S 16
 269 +#define READ_DATA_BYTE_NUM_S 11
 270 +
 271 +#define SNF_DLY_CTL3 0x548
 272 +#define SFCK_SAM_DLY_S 0
 273 +
 274 +#define SNF_STA_CTL1 0x550
 275 +#define CUS_PG_DONE BIT(28)
 276 +#define CUS_READ_DONE BIT(27)
 277 +#define SPI_STATE_S 0
 278 +#define SPI_STATE GENMASK(3, 0)
 279 +
 280 +#define SNF_CFG 0x55c
 281 +#define SPI_MODE BIT(0)
 282 +
 283 +#define SNF_GPRAM 0x800
 284 +#define SNF_GPRAM_SIZE 0xa0
 285 +
 286 +#define SNFI_POLL_INTERVAL 1000000
 287 +
 288 +static const u8 mt7622_spare_sizes[] = { 16, 26, 27, 28 };
 289 +
 290 +struct mtk_snand_caps {
 291 +       u16 sector_size;
 292 +       u16 max_sectors;
 293 +       u16 fdm_size;
 294 +       u16 fdm_ecc_size;
 295 +       u16 fifo_size;
 296 +
 297 +       bool bbm_swap;
 298 +       bool empty_page_check;
 299 +       u32 mastersta_mask;
 300 +
 301 +       const u8 *spare_sizes;
 302 +       u32 num_spare_size;
 303 +};
 304 +
 305 +static const struct mtk_snand_caps mt7622_snand_caps = {
 306 +       .sector_size = 512,
 307 +       .max_sectors = 8,
 308 +       .fdm_size = 8,
 309 +       .fdm_ecc_size = 1,
 310 +       .fifo_size = 32,
 311 +       .bbm_swap = false,
 312 +       .empty_page_check = false,
 313 +       .mastersta_mask = NFI_MASTERSTA_MASK_7622,
 314 +       .spare_sizes = mt7622_spare_sizes,
 315 +       .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
 316 +};
 317 +
 318 +static const struct mtk_snand_caps mt7629_snand_caps = {
 319 +       .sector_size = 512,
 320 +       .max_sectors = 8,
 321 +       .fdm_size = 8,
 322 +       .fdm_ecc_size = 1,
 323 +       .fifo_size = 32,
 324 +       .bbm_swap = true,
 325 +       .empty_page_check = false,
 326 +       .mastersta_mask = NFI_MASTERSTA_MASK_7622,
 327 +       .spare_sizes = mt7622_spare_sizes,
 328 +       .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
 329 +};
 330 +
 331 +struct mtk_snand_conf {
 332 +       size_t page_size;
 333 +       size_t oob_size;
 334 +       u8 nsectors;
 335 +       u8 spare_size;
 336 +};
 337 +
 338 +struct mtk_snand {
 339 +       struct spi_controller *ctlr;
 340 +       struct device *dev;
 341 +       struct clk *nfi_clk;
 342 +       struct clk *pad_clk;
 343 +       void __iomem *nfi_base;
 344 +       int irq;
 345 +       struct completion op_done;
 346 +       const struct mtk_snand_caps *caps;
 347 +       struct mtk_ecc_config *ecc_cfg;
 348 +       struct mtk_ecc *ecc;
 349 +       struct mtk_snand_conf nfi_cfg;
 350 +       struct mtk_ecc_stats ecc_stats;
 351 +       struct nand_ecc_engine ecc_eng;
 352 +       bool autofmt;
 353 +       u8 *buf;
 354 +       size_t buf_len;
 355 +};
 356 +
 357 +static struct mtk_snand *nand_to_mtk_snand(struct nand_device *nand)
 358 +{
 359 +       struct nand_ecc_engine *eng = nand->ecc.engine;
 360 +
 361 +       return container_of(eng, struct mtk_snand, ecc_eng);
 362 +}
 363 +
 364 +static inline int snand_prepare_bouncebuf(struct mtk_snand *snf, size_t size)
 365 +{
 366 +       if (snf->buf_len >= size)
 367 +               return 0;
 368 +       kfree(snf->buf);
 369 +       snf->buf = kmalloc(size, GFP_KERNEL);
 370 +       if (!snf->buf)
 371 +               return -ENOMEM;
 372 +       snf->buf_len = size;
 373 +       memset(snf->buf, 0xff, snf->buf_len);
 374 +       return 0;
 375 +}
 376 +
 377 +static inline u32 nfi_read32(struct mtk_snand *snf, u32 reg)
 378 +{
 379 +       return readl(snf->nfi_base + reg);
 380 +}
 381 +
 382 +static inline void nfi_write32(struct mtk_snand *snf, u32 reg, u32 val)
 383 +{
 384 +       writel(val, snf->nfi_base + reg);
 385 +}
 386 +
 387 +static inline void nfi_write16(struct mtk_snand *snf, u32 reg, u16 val)
 388 +{
 389 +       writew(val, snf->nfi_base + reg);
 390 +}
 391 +
 392 +static inline void nfi_rmw32(struct mtk_snand *snf, u32 reg, u32 clr, u32 set)
 393 +{
 394 +       u32 val;
 395 +
 396 +       val = readl(snf->nfi_base + reg);
 397 +       val &= ~clr;
 398 +       val |= set;
 399 +       writel(val, snf->nfi_base + reg);
 400 +}
 401 +
 402 +static void nfi_read_data(struct mtk_snand *snf, u32 reg, u8 *data, u32 len)
 403 +{
 404 +       u32 i, val = 0, es = sizeof(u32);
 405 +
 406 +       for (i = reg; i < reg + len; i++) {
 407 +               if (i == reg || i % es == 0)
 408 +                       val = nfi_read32(snf, i & ~(es - 1));
 409 +
 410 +               *data++ = (u8)(val >> (8 * (i % es)));
 411 +       }
 412 +}
 413 +
 414 +static int mtk_nfi_reset(struct mtk_snand *snf)
 415 +{
 416 +       u32 val, fifo_mask;
 417 +       int ret;
 418 +
 419 +       nfi_write32(snf, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST);
 420 +
 421 +       ret = readw_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
 422 +                                !(val & snf->caps->mastersta_mask), 0,
 423 +                                SNFI_POLL_INTERVAL);
 424 +       if (ret) {
 425 +               dev_err(snf->dev, "NFI master is still busy after reset\n");
 426 +               return ret;
 427 +       }
 428 +
 429 +       ret = readl_poll_timeout(snf->nfi_base + NFI_STA, val,
 430 +                                !(val & (NFI_FSM | NFI_NAND_FSM)), 0,
 431 +                                SNFI_POLL_INTERVAL);
 432 +       if (ret) {
 433 +               dev_err(snf->dev, "Failed to reset NFI\n");
 434 +               return ret;
 435 +       }
 436 +
 437 +       fifo_mask = ((snf->caps->fifo_size - 1) << FIFO_RD_REMAIN_S) |
 438 +                   ((snf->caps->fifo_size - 1) << FIFO_WR_REMAIN_S);
 439 +       ret = readw_poll_timeout(snf->nfi_base + NFI_FIFOSTA, val,
 440 +                                !(val & fifo_mask), 0, SNFI_POLL_INTERVAL);
 441 +       if (ret) {
 442 +               dev_err(snf->dev, "NFI FIFOs are not empty\n");
 443 +               return ret;
 444 +       }
 445 +
 446 +       return 0;
 447 +}
 448 +
 449 +static int mtk_snand_mac_reset(struct mtk_snand *snf)
 450 +{
 451 +       int ret;
 452 +       u32 val;
 453 +
 454 +       nfi_rmw32(snf, SNF_MISC_CTL, 0, SW_RST);
 455 +
 456 +       ret = readl_poll_timeout(snf->nfi_base + SNF_STA_CTL1, val,
 457 +                                !(val & SPI_STATE), 0, SNFI_POLL_INTERVAL);
 458 +       if (ret)
 459 +               dev_err(snf->dev, "Failed to reset SNFI MAC\n");
 460 +
 461 +       nfi_write32(snf, SNF_MISC_CTL,
 462 +                   (2 << FIFO_RD_LTC_S) | (10 << CS_DESELECT_CYC_S));
 463 +
 464 +       return ret;
 465 +}
 466 +
 467 +static int mtk_snand_mac_trigger(struct mtk_snand *snf, u32 outlen, u32 inlen)
 468 +{
 469 +       int ret;
 470 +       u32 val;
 471 +
 472 +       nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN);
 473 +       nfi_write32(snf, SNF_MAC_OUTL, outlen);
 474 +       nfi_write32(snf, SNF_MAC_INL, inlen);
 475 +
 476 +       nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN | SF_TRIG);
 477 +
 478 +       ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val,
 479 +                                val & WIP_READY, 0, SNFI_POLL_INTERVAL);
 480 +       if (ret) {
 481 +               dev_err(snf->dev, "Timed out waiting for WIP_READY\n");
 482 +               goto cleanup;
 483 +       }
 484 +
 485 +       ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val, !(val & WIP),
 486 +                                0, SNFI_POLL_INTERVAL);
 487 +       if (ret)
 488 +               dev_err(snf->dev, "Timed out waiting for WIP cleared\n");
 489 +
 490 +cleanup:
 491 +       nfi_write32(snf, SNF_MAC_CTL, 0);
 492 +
 493 +       return ret;
 494 +}
 495 +
 496 +static int mtk_snand_mac_io(struct mtk_snand *snf, const struct spi_mem_op *op)
 497 +{
 498 +       u32 rx_len = 0;
 499 +       u32 reg_offs = 0;
 500 +       u32 val = 0;
 501 +       const u8 *tx_buf = NULL;
 502 +       u8 *rx_buf = NULL;
 503 +       int i, ret;
 504 +       u8 b;
 505 +
 506 +       if (op->data.dir == SPI_MEM_DATA_IN) {
 507 +               rx_len = op->data.nbytes;
 508 +               rx_buf = op->data.buf.in;
 509 +       } else {
 510 +               tx_buf = op->data.buf.out;
 511 +       }
 512 +
 513 +       mtk_snand_mac_reset(snf);
 514 +
 515 +       for (i = 0; i < op->cmd.nbytes; i++, reg_offs++) {
 516 +               b = (op->cmd.opcode >> ((op->cmd.nbytes - i - 1) * 8)) & 0xff;
 517 +               val |= b << (8 * (reg_offs % 4));
 518 +               if (reg_offs % 4 == 3) {
 519 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 520 +                       val = 0;
 521 +               }
 522 +       }
 523 +
 524 +       for (i = 0; i < op->addr.nbytes; i++, reg_offs++) {
 525 +               b = (op->addr.val >> ((op->addr.nbytes - i - 1) * 8)) & 0xff;
 526 +               val |= b << (8 * (reg_offs % 4));
 527 +               if (reg_offs % 4 == 3) {
 528 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 529 +                       val = 0;
 530 +               }
 531 +       }
 532 +
 533 +       for (i = 0; i < op->dummy.nbytes; i++, reg_offs++) {
 534 +               if (reg_offs % 4 == 3) {
 535 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 536 +                       val = 0;
 537 +               }
 538 +       }
 539 +
 540 +       if (op->data.dir == SPI_MEM_DATA_OUT) {
 541 +               for (i = 0; i < op->data.nbytes; i++, reg_offs++) {
 542 +                       val |= tx_buf[i] << (8 * (reg_offs % 4));
 543 +                       if (reg_offs % 4 == 3) {
 544 +                               nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 545 +                               val = 0;
 546 +                       }
 547 +               }
 548 +       }
 549 +
 550 +       if (reg_offs % 4)
 551 +               nfi_write32(snf, SNF_GPRAM + (reg_offs & ~3), val);
 552 +
 553 +       for (i = 0; i < reg_offs; i += 4)
 554 +               dev_dbg(snf->dev, "%d: %08X", i,
 555 +                       nfi_read32(snf, SNF_GPRAM + i));
 556 +
 557 +       dev_dbg(snf->dev, "SNF TX: %u RX: %u", reg_offs, rx_len);
 558 +
 559 +       ret = mtk_snand_mac_trigger(snf, reg_offs, rx_len);
 560 +       if (ret)
 561 +               return ret;
 562 +
 563 +       if (!rx_len)
 564 +               return 0;
 565 +
 566 +       nfi_read_data(snf, SNF_GPRAM + reg_offs, rx_buf, rx_len);
 567 +       return 0;
 568 +}
 569 +
 570 +static int mtk_snand_setup_pagefmt(struct mtk_snand *snf, u32 page_size,
 571 +                                  u32 oob_size)
 572 +{
 573 +       int spare_idx = -1;
 574 +       u32 spare_size, spare_size_shift, pagesize_idx;
 575 +       u32 sector_size_512;
 576 +       u8 nsectors;
 577 +       int i;
 578 +
 579 +       // skip if it's already configured as required.
 580 +       if (snf->nfi_cfg.page_size == page_size &&
 581 +           snf->nfi_cfg.oob_size == oob_size)
 582 +               return 0;
 583 +
 584 +       nsectors = page_size / snf->caps->sector_size;
 585 +       if (nsectors > snf->caps->max_sectors) {
 586 +               dev_err(snf->dev, "too many sectors required.\n");
 587 +               goto err;
 588 +       }
 589 +
 590 +       if (snf->caps->sector_size == 512) {
 591 +               sector_size_512 = NFI_SEC_SEL_512;
 592 +               spare_size_shift = NFI_SPARE_SIZE_S;
 593 +       } else {
 594 +               sector_size_512 = 0;
 595 +               spare_size_shift = NFI_SPARE_SIZE_LS_S;
 596 +       }
 597 +
 598 +       switch (page_size) {
 599 +       case SZ_512:
 600 +               pagesize_idx = NFI_PAGE_SIZE_512_2K;
 601 +               break;
 602 +       case SZ_2K:
 603 +               if (snf->caps->sector_size == 512)
 604 +                       pagesize_idx = NFI_PAGE_SIZE_2K_4K;
 605 +               else
 606 +                       pagesize_idx = NFI_PAGE_SIZE_512_2K;
 607 +               break;
 608 +       case SZ_4K:
 609 +               if (snf->caps->sector_size == 512)
 610 +                       pagesize_idx = NFI_PAGE_SIZE_4K_8K;
 611 +               else
 612 +                       pagesize_idx = NFI_PAGE_SIZE_2K_4K;
 613 +               break;
 614 +       case SZ_8K:
 615 +               if (snf->caps->sector_size == 512)
 616 +                       pagesize_idx = NFI_PAGE_SIZE_8K_16K;
 617 +               else
 618 +                       pagesize_idx = NFI_PAGE_SIZE_4K_8K;
 619 +               break;
 620 +       case SZ_16K:
 621 +               pagesize_idx = NFI_PAGE_SIZE_8K_16K;
 622 +               break;
 623 +       default:
 624 +               dev_err(snf->dev, "unsupported page size.\n");
 625 +               goto err;
 626 +       }
 627 +
 628 +       spare_size = oob_size / nsectors;
 629 +       // If we're using the 1KB sector size, HW will automatically double the
 630 +       // spare size. We should only use half of the value in this case.
 631 +       if (snf->caps->sector_size == 1024)
 632 +               spare_size /= 2;
 633 +
 634 +       for (i = snf->caps->num_spare_size - 1; i >= 0; i--) {
 635 +               if (snf->caps->spare_sizes[i] <= spare_size) {
 636 +                       spare_size = snf->caps->spare_sizes[i];
 637 +                       if (snf->caps->sector_size == 1024)
 638 +                               spare_size *= 2;
 639 +                       spare_idx = i;
 640 +                       break;
 641 +               }
 642 +       }
 643 +
 644 +       if (spare_idx < 0) {
 645 +               dev_err(snf->dev, "unsupported spare size: %u\n", spare_size);
 646 +               goto err;
 647 +       }
 648 +
 649 +       nfi_write32(snf, NFI_PAGEFMT,
 650 +                   (snf->caps->fdm_ecc_size << NFI_FDM_ECC_NUM_S) |
 651 +                           (snf->caps->fdm_size << NFI_FDM_NUM_S) |
 652 +                           (spare_idx << spare_size_shift) |
 653 +                           (pagesize_idx << NFI_PAGE_SIZE_S) |
 654 +                           sector_size_512);
 655 +
 656 +       snf->nfi_cfg.page_size = page_size;
 657 +       snf->nfi_cfg.oob_size = oob_size;
 658 +       snf->nfi_cfg.nsectors = nsectors;
 659 +       snf->nfi_cfg.spare_size = spare_size;
 660 +
 661 +       dev_dbg(snf->dev, "page format: (%u + %u) * %u\n",
 662 +               snf->caps->sector_size, spare_size, nsectors);
 663 +       return snand_prepare_bouncebuf(snf, page_size + oob_size);
 664 +err:
 665 +       dev_err(snf->dev, "page size %u + %u is not supported\n", page_size,
 666 +               oob_size);
 667 +       return -EOPNOTSUPP;
 668 +}
 669 +
 670 +static int mtk_snand_ooblayout_ecc(struct mtd_info *mtd, int section,
 671 +                                  struct mtd_oob_region *oobecc)
 672 +{
 673 +       // ECC area is not accessible
 674 +       return -ERANGE;
 675 +}
 676 +
 677 +static int mtk_snand_ooblayout_free(struct mtd_info *mtd, int section,
 678 +                                   struct mtd_oob_region *oobfree)
 679 +{
 680 +       struct nand_device *nand = mtd_to_nanddev(mtd);
 681 +       struct mtk_snand *ms = nand_to_mtk_snand(nand);
 682 +
 683 +       if (section >= ms->nfi_cfg.nsectors)
 684 +               return -ERANGE;
 685 +
 686 +       oobfree->length = ms->caps->fdm_size - 1;
 687 +       oobfree->offset = section * ms->caps->fdm_size + 1;
 688 +       return 0;
 689 +}
 690 +
 691 +static const struct mtd_ooblayout_ops mtk_snand_ooblayout = {
 692 +       .ecc = mtk_snand_ooblayout_ecc,
 693 +       .free = mtk_snand_ooblayout_free,
 694 +};
 695 +
 696 +static int mtk_snand_ecc_init_ctx(struct nand_device *nand)
 697 +{
 698 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 699 +       struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
 700 +       struct nand_ecc_props *reqs = &nand->ecc.requirements;
 701 +       struct nand_ecc_props *user = &nand->ecc.user_conf;
 702 +       struct mtd_info *mtd = nanddev_to_mtd(nand);
 703 +       int step_size = 0, strength = 0, desired_correction = 0, steps;
 704 +       bool ecc_user = false;
 705 +       int ret;
 706 +       u32 parity_bits, max_ecc_bytes;
 707 +       struct mtk_ecc_config *ecc_cfg;
 708 +
 709 +       ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
 710 +                                     nand->memorg.oobsize);
 711 +       if (ret)
 712 +               return ret;
 713 +
 714 +       ecc_cfg = kzalloc(sizeof(*ecc_cfg), GFP_KERNEL);
 715 +       if (!ecc_cfg)
 716 +               return -ENOMEM;
 717 +
 718 +       nand->ecc.ctx.priv = ecc_cfg;
 719 +
 720 +       if (user->step_size && user->strength) {
 721 +               step_size = user->step_size;
 722 +               strength = user->strength;
 723 +               ecc_user = true;
 724 +       } else if (reqs->step_size && reqs->strength) {
 725 +               step_size = reqs->step_size;
 726 +               strength = reqs->strength;
 727 +       }
 728 +
 729 +       if (step_size && strength) {
 730 +               steps = mtd->writesize / step_size;
 731 +               desired_correction = steps * strength;
 732 +               strength = desired_correction / snf->nfi_cfg.nsectors;
 733 +       }
 734 +
 735 +       ecc_cfg->mode = ECC_NFI_MODE;
 736 +       ecc_cfg->sectors = snf->nfi_cfg.nsectors;
 737 +       ecc_cfg->len = snf->caps->sector_size + snf->caps->fdm_ecc_size;
 738 +
 739 +       // calculate the max possible strength under current page format
 740 +       parity_bits = mtk_ecc_get_parity_bits(snf->ecc);
 741 +       max_ecc_bytes = snf->nfi_cfg.spare_size - snf->caps->fdm_size;
 742 +       ecc_cfg->strength = max_ecc_bytes * 8 / parity_bits;
 743 +       mtk_ecc_adjust_strength(snf->ecc, &ecc_cfg->strength);
 744 +
 745 +       // if there's a user requested strength, find the minimum strength that
 746 +       // meets the requirement. Otherwise use the maximum strength which is
 747 +       // expected by BootROM.
 748 +       if (ecc_user && strength) {
 749 +               u32 s_next = ecc_cfg->strength - 1;
 750 +
 751 +               while (1) {
 752 +                       mtk_ecc_adjust_strength(snf->ecc, &s_next);
 753 +                       if (s_next >= ecc_cfg->strength)
 754 +                               break;
 755 +                       if (s_next < strength)
 756 +                               break;
 757 +                       s_next = ecc_cfg->strength - 1;
 758 +               }
 759 +       }
 760 +
 761 +       mtd_set_ooblayout(mtd, &mtk_snand_ooblayout);
 762 +
 763 +       conf->step_size = snf->caps->sector_size;
 764 +       conf->strength = ecc_cfg->strength;
 765 +
 766 +       if (ecc_cfg->strength < strength)
 767 +               dev_warn(snf->dev, "unable to fulfill ECC of %u bits.\n",
 768 +                        strength);
 769 +       dev_info(snf->dev, "ECC strength: %u bits per %u bytes\n",
 770 +                ecc_cfg->strength, snf->caps->sector_size);
 771 +
 772 +       return 0;
 773 +}
 774 +
 775 +static void mtk_snand_ecc_cleanup_ctx(struct nand_device *nand)
 776 +{
 777 +       struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
 778 +
 779 +       kfree(ecc_cfg);
 780 +}
 781 +
 782 +static int mtk_snand_ecc_prepare_io_req(struct nand_device *nand,
 783 +                                       struct nand_page_io_req *req)
 784 +{
 785 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 786 +       struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
 787 +       int ret;
 788 +
 789 +       ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
 790 +                                     nand->memorg.oobsize);
 791 +       if (ret)
 792 +               return ret;
 793 +       snf->autofmt = true;
 794 +       snf->ecc_cfg = ecc_cfg;
 795 +       return 0;
 796 +}
 797 +
 798 +static int mtk_snand_ecc_finish_io_req(struct nand_device *nand,
 799 +                                      struct nand_page_io_req *req)
 800 +{
 801 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 802 +       struct mtd_info *mtd = nanddev_to_mtd(nand);
 803 +
 804 +       snf->ecc_cfg = NULL;
 805 +       snf->autofmt = false;
 806 +       if ((req->mode == MTD_OPS_RAW) || (req->type != NAND_PAGE_READ))
 807 +               return 0;
 808 +
 809 +       if (snf->ecc_stats.failed)
 810 +               mtd->ecc_stats.failed += snf->ecc_stats.failed;
 811 +       mtd->ecc_stats.corrected += snf->ecc_stats.corrected;
 812 +       return snf->ecc_stats.failed ? -EBADMSG : snf->ecc_stats.bitflips;
 813 +}
 814 +
 815 +static struct nand_ecc_engine_ops mtk_snfi_ecc_engine_ops = {
 816 +       .init_ctx = mtk_snand_ecc_init_ctx,
 817 +       .cleanup_ctx = mtk_snand_ecc_cleanup_ctx,
 818 +       .prepare_io_req = mtk_snand_ecc_prepare_io_req,
 819 +       .finish_io_req = mtk_snand_ecc_finish_io_req,
 820 +};
 821 +
 822 +static void mtk_snand_read_fdm(struct mtk_snand *snf, u8 *buf)
 823 +{
 824 +       u32 vall, valm;
 825 +       u8 *oobptr = buf;
 826 +       int i, j;
 827 +
 828 +       for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
 829 +               vall = nfi_read32(snf, NFI_FDML(i));
 830 +               valm = nfi_read32(snf, NFI_FDMM(i));
 831 +
 832 +               for (j = 0; j < snf->caps->fdm_size; j++)
 833 +                       oobptr[j] = (j >= 4 ? valm : vall) >> ((j % 4) * 8);
 834 +
 835 +               oobptr += snf->caps->fdm_size;
 836 +       }
 837 +}
 838 +
 839 +static void mtk_snand_write_fdm(struct mtk_snand *snf, const u8 *buf)
 840 +{
 841 +       u32 fdm_size = snf->caps->fdm_size;
 842 +       const u8 *oobptr = buf;
 843 +       u32 vall, valm;
 844 +       int i, j;
 845 +
 846 +       for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
 847 +               vall = 0;
 848 +               valm = 0;
 849 +
 850 +               for (j = 0; j < 8; j++) {
 851 +                       if (j < 4)
 852 +                               vall |= (j < fdm_size ? oobptr[j] : 0xff)
 853 +                                       << (j * 8);
 854 +                       else
 855 +                               valm |= (j < fdm_size ? oobptr[j] : 0xff)
 856 +                                       << ((j - 4) * 8);
 857 +               }
 858 +
 859 +               nfi_write32(snf, NFI_FDML(i), vall);
 860 +               nfi_write32(snf, NFI_FDMM(i), valm);
 861 +
 862 +               oobptr += fdm_size;
 863 +       }
 864 +}
 865 +
 866 +static void mtk_snand_bm_swap(struct mtk_snand *snf, u8 *buf)
 867 +{
 868 +       u32 buf_bbm_pos, fdm_bbm_pos;
 869 +
 870 +       if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
 871 +               return;
 872 +
 873 +       // swap [pagesize] byte on nand with the first fdm byte
 874 +       // in the last sector.
 875 +       buf_bbm_pos = snf->nfi_cfg.page_size -
 876 +                     (snf->nfi_cfg.nsectors - 1) * snf->nfi_cfg.spare_size;
 877 +       fdm_bbm_pos = snf->nfi_cfg.page_size +
 878 +                     (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
 879 +
 880 +       swap(snf->buf[fdm_bbm_pos], buf[buf_bbm_pos]);
 881 +}
 882 +
 883 +static void mtk_snand_fdm_bm_swap(struct mtk_snand *snf)
 884 +{
 885 +       u32 fdm_bbm_pos1, fdm_bbm_pos2;
 886 +
 887 +       if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
 888 +               return;
 889 +
 890 +       // swap the first fdm byte in the first and the last sector.
 891 +       fdm_bbm_pos1 = snf->nfi_cfg.page_size;
 892 +       fdm_bbm_pos2 = snf->nfi_cfg.page_size +
 893 +                      (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
 894 +       swap(snf->buf[fdm_bbm_pos1], snf->buf[fdm_bbm_pos2]);
 895 +}
 896 +
 897 +static int mtk_snand_read_page_cache(struct mtk_snand *snf,
 898 +                                    const struct spi_mem_op *op)
 899 +{
 900 +       u8 *buf = snf->buf;
 901 +       u8 *buf_fdm = buf + snf->nfi_cfg.page_size;
 902 +       // the address part to be sent by the controller
 903 +       u32 op_addr = op->addr.val;
 904 +       // where to start copying data from bounce buffer
 905 +       u32 rd_offset = 0;
 906 +       u32 dummy_clk = (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth);
 907 +       u32 op_mode = 0;
 908 +       u32 dma_len = snf->buf_len;
 909 +       int ret = 0;
 910 +       u32 rd_mode, rd_bytes, val;
 911 +       dma_addr_t buf_dma;
 912 +
 913 +       if (snf->autofmt) {
 914 +               u32 last_bit;
 915 +               u32 mask;
 916 +
 917 +               dma_len = snf->nfi_cfg.page_size;
 918 +               op_mode = CNFG_AUTO_FMT_EN;
 919 +               if (op->data.ecc)
 920 +                       op_mode |= CNFG_HW_ECC_EN;
 921 +               // extract the plane bit:
 922 +               // Find the highest bit set in (pagesize+oobsize).
 923 +               // Bits higher than that in op->addr are kept and sent over SPI
 924 +               // Lower bits are used as an offset for copying data from DMA
 925 +               // bounce buffer.
 926 +               last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
 927 +               mask = (1 << last_bit) - 1;
 928 +               rd_offset = op_addr & mask;
 929 +               op_addr &= ~mask;
 930 +
 931 +               // check if we can dma to the caller memory
 932 +               if (rd_offset == 0 && op->data.nbytes >= snf->nfi_cfg.page_size)
 933 +                       buf = op->data.buf.in;
 934 +       }
 935 +       mtk_snand_mac_reset(snf);
 936 +       mtk_nfi_reset(snf);
 937 +
 938 +       // command and dummy cycles
 939 +       nfi_write32(snf, SNF_RD_CTL2,
 940 +                   (dummy_clk << DATA_READ_DUMMY_S) |
 941 +                           (op->cmd.opcode << DATA_READ_CMD_S));
 942 +
 943 +       // read address
 944 +       nfi_write32(snf, SNF_RD_CTL3, op_addr);
 945 +
 946 +       // Set read op_mode
 947 +       if (op->data.buswidth == 4)
 948 +               rd_mode = op->addr.buswidth == 4 ? DATA_READ_MODE_QUAD :
 949 +                                                  DATA_READ_MODE_X4;
 950 +       else if (op->data.buswidth == 2)
 951 +               rd_mode = op->addr.buswidth == 2 ? DATA_READ_MODE_DUAL :
 952 +                                                  DATA_READ_MODE_X2;
 953 +       else
 954 +               rd_mode = DATA_READ_MODE_X1;
 955 +       rd_mode <<= DATA_READ_MODE_S;
 956 +       nfi_rmw32(snf, SNF_MISC_CTL, DATA_READ_MODE,
 957 +                 rd_mode | DATARD_CUSTOM_EN);
 958 +
 959 +       // Set bytes to read
 960 +       rd_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
 961 +                  snf->nfi_cfg.nsectors;
 962 +       nfi_write32(snf, SNF_MISC_CTL2,
 963 +                   (rd_bytes << PROGRAM_LOAD_BYTE_NUM_S) | rd_bytes);
 964 +
 965 +       // NFI read prepare
 966 +       nfi_write16(snf, NFI_CNFG,
 967 +                   (CNFG_OP_MODE_CUST << CNFG_OP_MODE_S) | CNFG_DMA_BURST_EN |
 968 +                           CNFG_READ_MODE | CNFG_DMA_MODE | op_mode);
 969 +
 970 +       nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
 971 +
 972 +       buf_dma = dma_map_single(snf->dev, buf, dma_len, DMA_FROM_DEVICE);
 973 +       if (dma_mapping_error(snf->dev, buf_dma)) {
 974 +               dev_err(snf->dev, "DMA mapping failed.\n");
 975 +               goto cleanup;
 976 +       }
 977 +       nfi_write32(snf, NFI_STRADDR, buf_dma);
 978 +       if (op->data.ecc) {
 979 +               snf->ecc_cfg->op = ECC_DECODE;
 980 +               ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
 981 +               if (ret)
 982 +                       goto cleanup_dma;
 983 +       }
 984 +       // Prepare for custom read interrupt
 985 +       nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_READ);
 986 +       reinit_completion(&snf->op_done);
 987 +
 988 +       // Trigger NFI into custom mode
 989 +       nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_READ);
 990 +
 991 +       // Start DMA read
 992 +       nfi_rmw32(snf, NFI_CON, 0, CON_BRD);
 993 +       nfi_write16(snf, NFI_STRDATA, STR_DATA);
 994 +
 995 +       if (!wait_for_completion_timeout(
 996 +                   &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
 997 +               dev_err(snf->dev, "DMA timed out for reading from cache.\n");
 998 +               ret = -ETIMEDOUT;
 999 +               goto cleanup;
1000 +       }
1001 +
1002 +       // Wait for BUS_SEC_CNTR returning expected value
1003 +       ret = readl_poll_timeout(snf->nfi_base + NFI_BYTELEN, val,
1004 +                                BUS_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1005 +                                SNFI_POLL_INTERVAL);
1006 +       if (ret) {
1007 +               dev_err(snf->dev, "Timed out waiting for BUS_SEC_CNTR\n");
1008 +               goto cleanup2;
1009 +       }
1010 +
1011 +       // Wait for bus becoming idle
1012 +       ret = readl_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
1013 +                                !(val & snf->caps->mastersta_mask), 0,
1014 +                                SNFI_POLL_INTERVAL);
1015 +       if (ret) {
1016 +               dev_err(snf->dev, "Timed out waiting for bus becoming idle\n");
1017 +               goto cleanup2;
1018 +       }
1019 +
1020 +       if (op->data.ecc) {
1021 +               ret = mtk_ecc_wait_done(snf->ecc, ECC_DECODE);
1022 +               if (ret) {
1023 +                       dev_err(snf->dev, "wait ecc done timeout\n");
1024 +                       goto cleanup2;
1025 +               }
1026 +               // save status before disabling ecc
1027 +               mtk_ecc_get_stats(snf->ecc, &snf->ecc_stats,
1028 +                                 snf->nfi_cfg.nsectors);
1029 +       }
1030 +
1031 +       dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1032 +
1033 +       if (snf->autofmt) {
1034 +               mtk_snand_read_fdm(snf, buf_fdm);
1035 +               if (snf->caps->bbm_swap) {
1036 +                       mtk_snand_bm_swap(snf, buf);
1037 +                       mtk_snand_fdm_bm_swap(snf);
1038 +               }
1039 +       }
1040 +
1041 +       // copy data back
1042 +       if (nfi_read32(snf, NFI_STA) & READ_EMPTY) {
1043 +               memset(op->data.buf.in, 0xff, op->data.nbytes);
1044 +               snf->ecc_stats.bitflips = 0;
1045 +               snf->ecc_stats.failed = 0;
1046 +               snf->ecc_stats.corrected = 0;
1047 +       } else {
1048 +               if (buf == op->data.buf.in) {
1049 +                       u32 cap_len = snf->buf_len - snf->nfi_cfg.page_size;
1050 +                       u32 req_left = op->data.nbytes - snf->nfi_cfg.page_size;
1051 +
1052 +                       if (req_left)
1053 +                               memcpy(op->data.buf.in + snf->nfi_cfg.page_size,
1054 +                                      buf_fdm,
1055 +                                      cap_len < req_left ? cap_len : req_left);
1056 +               } else if (rd_offset < snf->buf_len) {
1057 +                       u32 cap_len = snf->buf_len - rd_offset;
1058 +
1059 +                       if (op->data.nbytes < cap_len)
1060 +                               cap_len = op->data.nbytes;
1061 +                       memcpy(op->data.buf.in, snf->buf + rd_offset, cap_len);
1062 +               }
1063 +       }
1064 +cleanup2:
1065 +       if (op->data.ecc)
1066 +               mtk_ecc_disable(snf->ecc);
1067 +cleanup_dma:
1068 +       // unmap dma only if any error happens. (otherwise it's done before
1069 +       // data copying)
1070 +       if (ret)
1071 +               dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1072 +cleanup:
1073 +       // Stop read
1074 +       nfi_write32(snf, NFI_CON, 0);
1075 +       nfi_write16(snf, NFI_CNFG, 0);
1076 +
1077 +       // Clear SNF done flag
1078 +       nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_READ_DONE);
1079 +       nfi_write32(snf, SNF_STA_CTL1, 0);
1080 +
1081 +       // Disable interrupt
1082 +       nfi_read32(snf, NFI_INTR_STA);
1083 +       nfi_write32(snf, NFI_INTR_EN, 0);
1084 +
1085 +       nfi_rmw32(snf, SNF_MISC_CTL, DATARD_CUSTOM_EN, 0);
1086 +       return ret;
1087 +}
1088 +
1089 +static int mtk_snand_write_page_cache(struct mtk_snand *snf,
1090 +                                     const struct spi_mem_op *op)
1091 +{
1092 +       // the address part to be sent by the controller
1093 +       u32 op_addr = op->addr.val;
1094 +       // where to start copying data from bounce buffer
1095 +       u32 wr_offset = 0;
1096 +       u32 op_mode = 0;
1097 +       int ret = 0;
1098 +       u32 wr_mode = 0;
1099 +       u32 dma_len = snf->buf_len;
1100 +       u32 wr_bytes, val;
1101 +       size_t cap_len;
1102 +       dma_addr_t buf_dma;
1103 +
1104 +       if (snf->autofmt) {
1105 +               u32 last_bit;
1106 +               u32 mask;
1107 +
1108 +               dma_len = snf->nfi_cfg.page_size;
1109 +               op_mode = CNFG_AUTO_FMT_EN;
1110 +               if (op->data.ecc)
1111 +                       op_mode |= CNFG_HW_ECC_EN;
1112 +
1113 +               last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
1114 +               mask = (1 << last_bit) - 1;
1115 +               wr_offset = op_addr & mask;
1116 +               op_addr &= ~mask;
1117 +       }
1118 +       mtk_snand_mac_reset(snf);
1119 +       mtk_nfi_reset(snf);
1120 +
1121 +       if (wr_offset)
1122 +               memset(snf->buf, 0xff, wr_offset);
1123 +
1124 +       cap_len = snf->buf_len - wr_offset;
1125 +       if (op->data.nbytes < cap_len)
1126 +               cap_len = op->data.nbytes;
1127 +       memcpy(snf->buf + wr_offset, op->data.buf.out, cap_len);
1128 +       if (snf->autofmt) {
1129 +               if (snf->caps->bbm_swap) {
1130 +                       mtk_snand_fdm_bm_swap(snf);
1131 +                       mtk_snand_bm_swap(snf, snf->buf);
1132 +               }
1133 +               mtk_snand_write_fdm(snf, snf->buf + snf->nfi_cfg.page_size);
1134 +       }
1135 +
1136 +       // Command
1137 +       nfi_write32(snf, SNF_PG_CTL1, (op->cmd.opcode << PG_LOAD_CMD_S));
1138 +
1139 +       // write address
1140 +       nfi_write32(snf, SNF_PG_CTL2, op_addr);
1141 +
1142 +       // Set read op_mode
1143 +       if (op->data.buswidth == 4)
1144 +               wr_mode = PG_LOAD_X4_EN;
1145 +
1146 +       nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_X4_EN,
1147 +                 wr_mode | PG_LOAD_CUSTOM_EN);
1148 +
1149 +       // Set bytes to write
1150 +       wr_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
1151 +                  snf->nfi_cfg.nsectors;
1152 +       nfi_write32(snf, SNF_MISC_CTL2,
1153 +                   (wr_bytes << PROGRAM_LOAD_BYTE_NUM_S) | wr_bytes);
1154 +
1155 +       // NFI write prepare
1156 +       nfi_write16(snf, NFI_CNFG,
1157 +                   (CNFG_OP_MODE_PROGRAM << CNFG_OP_MODE_S) |
1158 +                           CNFG_DMA_BURST_EN | CNFG_DMA_MODE | op_mode);
1159 +
1160 +       nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
1161 +       buf_dma = dma_map_single(snf->dev, snf->buf, dma_len, DMA_TO_DEVICE);
1162 +       if (dma_mapping_error(snf->dev, buf_dma)) {
1163 +               dev_err(snf->dev, "DMA mapping failed.\n");
1164 +               goto cleanup;
1165 +       }
1166 +       nfi_write32(snf, NFI_STRADDR, buf_dma);
1167 +       if (op->data.ecc) {
1168 +               snf->ecc_cfg->op = ECC_ENCODE;
1169 +               ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
1170 +               if (ret)
1171 +                       goto cleanup_dma;
1172 +       }
1173 +       // Prepare for custom write interrupt
1174 +       nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_PG);
1175 +       reinit_completion(&snf->op_done);
1176 +       ;
1177 +
1178 +       // Trigger NFI into custom mode
1179 +       nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_WRITE);
1180 +
1181 +       // Start DMA write
1182 +       nfi_rmw32(snf, NFI_CON, 0, CON_BWR);
1183 +       nfi_write16(snf, NFI_STRDATA, STR_DATA);
1184 +
1185 +       if (!wait_for_completion_timeout(
1186 +                   &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
1187 +               dev_err(snf->dev, "DMA timed out for program load.\n");
1188 +               ret = -ETIMEDOUT;
1189 +               goto cleanup_ecc;
1190 +       }
1191 +
1192 +       // Wait for NFI_SEC_CNTR returning expected value
1193 +       ret = readl_poll_timeout(snf->nfi_base + NFI_ADDRCNTR, val,
1194 +                                NFI_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1195 +                                SNFI_POLL_INTERVAL);
1196 +       if (ret)
1197 +               dev_err(snf->dev, "Timed out waiting for NFI_SEC_CNTR\n");
1198 +
1199 +cleanup_ecc:
1200 +       if (op->data.ecc)
1201 +               mtk_ecc_disable(snf->ecc);
1202 +cleanup_dma:
1203 +       dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_TO_DEVICE);
1204 +cleanup:
1205 +       // Stop write
1206 +       nfi_write32(snf, NFI_CON, 0);
1207 +       nfi_write16(snf, NFI_CNFG, 0);
1208 +
1209 +       // Clear SNF done flag
1210 +       nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_PG_DONE);
1211 +       nfi_write32(snf, SNF_STA_CTL1, 0);
1212 +
1213 +       // Disable interrupt
1214 +       nfi_read32(snf, NFI_INTR_STA);
1215 +       nfi_write32(snf, NFI_INTR_EN, 0);
1216 +
1217 +       nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_CUSTOM_EN, 0);
1218 +
1219 +       return ret;
1220 +}
1221 +
1222 +/**
1223 + * mtk_snand_is_page_ops() - check if the op is a controller supported page op.
1224 + * @op spi-mem op to check
1225 + *
1226 + * Check whether op can be executed with read_from_cache or program_load
1227 + * mode in the controller.
1228 + * This controller can execute typical Read From Cache and Program Load
1229 + * instructions found on SPI-NAND with 2-byte address.
1230 + * DTR and cmd buswidth & nbytes should be checked before calling this.
1231 + *
1232 + * Return: true if the op matches the instruction template
1233 + */
1234 +static bool mtk_snand_is_page_ops(const struct spi_mem_op *op)
1235 +{
1236 +       if (op->addr.nbytes != 2)
1237 +               return false;
1238 +
1239 +       if (op->addr.buswidth != 1 && op->addr.buswidth != 2 &&
1240 +           op->addr.buswidth != 4)
1241 +               return false;
1242 +
1243 +       // match read from page instructions
1244 +       if (op->data.dir == SPI_MEM_DATA_IN) {
1245 +               // check dummy cycle first
1246 +               if (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth >
1247 +                   DATA_READ_MAX_DUMMY)
1248 +                       return false;
1249 +               // quad io / quad out
1250 +               if ((op->addr.buswidth == 4 || op->addr.buswidth == 1) &&
1251 +                   op->data.buswidth == 4)
1252 +                       return true;
1253 +
1254 +               // dual io / dual out
1255 +               if ((op->addr.buswidth == 2 || op->addr.buswidth == 1) &&
1256 +                   op->data.buswidth == 2)
1257 +                       return true;
1258 +
1259 +               // standard spi
1260 +               if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1261 +                       return true;
1262 +       } else if (op->data.dir == SPI_MEM_DATA_OUT) {
1263 +               // check dummy cycle first
1264 +               if (op->dummy.nbytes)
1265 +                       return false;
1266 +               // program load quad out
1267 +               if (op->addr.buswidth == 1 && op->data.buswidth == 4)
1268 +                       return true;
1269 +               // standard spi
1270 +               if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1271 +                       return true;
1272 +       }
1273 +       return false;
1274 +}
1275 +
1276 +static bool mtk_snand_supports_op(struct spi_mem *mem,
1277 +                                 const struct spi_mem_op *op)
1278 +{
1279 +       if (!spi_mem_default_supports_op(mem, op))
1280 +               return false;
1281 +       if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1)
1282 +               return false;
1283 +       if (mtk_snand_is_page_ops(op))
1284 +               return true;
1285 +       return ((op->addr.nbytes == 0 || op->addr.buswidth == 1) &&
1286 +               (op->dummy.nbytes == 0 || op->dummy.buswidth == 1) &&
1287 +               (op->data.nbytes == 0 || op->data.buswidth == 1));
1288 +}
1289 +
1290 +static int mtk_snand_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
1291 +{
1292 +       struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1293 +       // page ops transfer size must be exactly ((sector_size + spare_size) *
1294 +       // nsectors). Limit the op size if the caller requests more than that.
1295 +       // exec_op will read more than needed and discard the leftover if the
1296 +       // caller requests less data.
1297 +       if (mtk_snand_is_page_ops(op)) {
1298 +               size_t l;
1299 +               // skip adjust_op_size for page ops
1300 +               if (ms->autofmt)
1301 +                       return 0;
1302 +               l = ms->caps->sector_size + ms->nfi_cfg.spare_size;
1303 +               l *= ms->nfi_cfg.nsectors;
1304 +               if (op->data.nbytes > l)
1305 +                       op->data.nbytes = l;
1306 +       } else {
1307 +               size_t hl = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
1308 +
1309 +               if (hl >= SNF_GPRAM_SIZE)
1310 +                       return -EOPNOTSUPP;
1311 +               if (op->data.nbytes > SNF_GPRAM_SIZE - hl)
1312 +                       op->data.nbytes = SNF_GPRAM_SIZE - hl;
1313 +       }
1314 +       return 0;
1315 +}
1316 +
1317 +static int mtk_snand_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
1318 +{
1319 +       struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1320 +
1321 +       dev_dbg(ms->dev, "OP %02x ADDR %08llX@%d:%u DATA %d:%u", op->cmd.opcode,
1322 +               op->addr.val, op->addr.buswidth, op->addr.nbytes,
1323 +               op->data.buswidth, op->data.nbytes);
1324 +       if (mtk_snand_is_page_ops(op)) {
1325 +               if (op->data.dir == SPI_MEM_DATA_IN)
1326 +                       return mtk_snand_read_page_cache(ms, op);
1327 +               else
1328 +                       return mtk_snand_write_page_cache(ms, op);
1329 +       } else {
1330 +               return mtk_snand_mac_io(ms, op);
1331 +       }
1332 +}
1333 +
1334 +static const struct spi_controller_mem_ops mtk_snand_mem_ops = {
1335 +       .adjust_op_size = mtk_snand_adjust_op_size,
1336 +       .supports_op = mtk_snand_supports_op,
1337 +       .exec_op = mtk_snand_exec_op,
1338 +};
1339 +
1340 +static const struct spi_controller_mem_caps mtk_snand_mem_caps = {
1341 +       .ecc = true,
1342 +};
1343 +
1344 +static irqreturn_t mtk_snand_irq(int irq, void *id)
1345 +{
1346 +       struct mtk_snand *snf = id;
1347 +       u32 sta, ien;
1348 +
1349 +       sta = nfi_read32(snf, NFI_INTR_STA);
1350 +       ien = nfi_read32(snf, NFI_INTR_EN);
1351 +
1352 +       if (!(sta & ien))
1353 +               return IRQ_NONE;
1354 +
1355 +       nfi_write32(snf, NFI_INTR_EN, 0);
1356 +       complete(&snf->op_done);
1357 +       return IRQ_HANDLED;
1358 +}
1359 +
1360 +static const struct of_device_id mtk_snand_ids[] = {
1361 +       { .compatible = "mediatek,mt7622-snand", .data = &mt7622_snand_caps },
1362 +       { .compatible = "mediatek,mt7629-snand", .data = &mt7629_snand_caps },
1363 +       {},
1364 +};
1365 +
1366 +MODULE_DEVICE_TABLE(of, mtk_snand_ids);
1367 +
1368 +static int mtk_snand_enable_clk(struct mtk_snand *ms)
1369 +{
1370 +       int ret;
1371 +
1372 +       ret = clk_prepare_enable(ms->nfi_clk);
1373 +       if (ret) {
1374 +               dev_err(ms->dev, "unable to enable nfi clk\n");
1375 +               return ret;
1376 +       }
1377 +       ret = clk_prepare_enable(ms->pad_clk);
1378 +       if (ret) {
1379 +               dev_err(ms->dev, "unable to enable pad clk\n");
1380 +               goto err1;
1381 +       }
1382 +       return 0;
1383 +err1:
1384 +       clk_disable_unprepare(ms->nfi_clk);
1385 +       return ret;
1386 +}
1387 +
1388 +static void mtk_snand_disable_clk(struct mtk_snand *ms)
1389 +{
1390 +       clk_disable_unprepare(ms->pad_clk);
1391 +       clk_disable_unprepare(ms->nfi_clk);
1392 +}
1393 +
1394 +static int mtk_snand_probe(struct platform_device *pdev)
1395 +{
1396 +       struct device_node *np = pdev->dev.of_node;
1397 +       const struct of_device_id *dev_id;
1398 +       struct spi_controller *ctlr;
1399 +       struct mtk_snand *ms;
1400 +       int ret;
1401 +
1402 +       dev_id = of_match_node(mtk_snand_ids, np);
1403 +       if (!dev_id)
1404 +               return -EINVAL;
1405 +
1406 +       ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*ms));
1407 +       if (!ctlr)
1408 +               return -ENOMEM;
1409 +       platform_set_drvdata(pdev, ctlr);
1410 +
1411 +       ms = spi_controller_get_devdata(ctlr);
1412 +
1413 +       ms->ctlr = ctlr;
1414 +       ms->caps = dev_id->data;
1415 +
1416 +       ms->ecc = of_mtk_ecc_get(np);
1417 +       if (IS_ERR(ms->ecc))
1418 +               return PTR_ERR(ms->ecc);
1419 +       else if (!ms->ecc)
1420 +               return -ENODEV;
1421 +
1422 +       ms->nfi_base = devm_platform_ioremap_resource(pdev, 0);
1423 +       if (IS_ERR(ms->nfi_base)) {
1424 +               ret = PTR_ERR(ms->nfi_base);
1425 +               goto release_ecc;
1426 +       }
1427 +
1428 +       ms->dev = &pdev->dev;
1429 +
1430 +       ms->nfi_clk = devm_clk_get(&pdev->dev, "nfi_clk");
1431 +       if (IS_ERR(ms->nfi_clk)) {
1432 +               ret = PTR_ERR(ms->nfi_clk);
1433 +               dev_err(&pdev->dev, "unable to get nfi_clk, err = %d\n", ret);
1434 +               goto release_ecc;
1435 +       }
1436 +
1437 +       ms->pad_clk = devm_clk_get(&pdev->dev, "pad_clk");
1438 +       if (IS_ERR(ms->pad_clk)) {
1439 +               ret = PTR_ERR(ms->pad_clk);
1440 +               dev_err(&pdev->dev, "unable to get pad_clk, err = %d\n", ret);
1441 +               goto release_ecc;
1442 +       }
1443 +
1444 +       ret = mtk_snand_enable_clk(ms);
1445 +       if (ret)
1446 +               goto release_ecc;
1447 +
1448 +       init_completion(&ms->op_done);
1449 +
1450 +       ms->irq = platform_get_irq(pdev, 0);
1451 +       if (ms->irq < 0) {
1452 +               ret = ms->irq;
1453 +               goto disable_clk;
1454 +       }
1455 +       ret = devm_request_irq(ms->dev, ms->irq, mtk_snand_irq, 0x0,
1456 +                              "mtk-snand", ms);
1457 +       if (ret) {
1458 +               dev_err(ms->dev, "failed to request snfi irq\n");
1459 +               goto disable_clk;
1460 +       }
1461 +
1462 +       ret = dma_set_mask(ms->dev, DMA_BIT_MASK(32));
1463 +       if (ret) {
1464 +               dev_err(ms->dev, "failed to set dma mask\n");
1465 +               goto disable_clk;
1466 +       }
1467 +
1468 +       // switch to SNFI mode
1469 +       nfi_write32(ms, SNF_CFG, SPI_MODE);
1470 +
1471 +       // setup an initial page format for ops matching page_cache_op template
1472 +       // before ECC is called.
1473 +       ret = mtk_snand_setup_pagefmt(ms, ms->caps->sector_size,
1474 +                                     ms->caps->spare_sizes[0]);
1475 +       if (ret) {
1476 +               dev_err(ms->dev, "failed to set initial page format\n");
1477 +               goto disable_clk;
1478 +       }
1479 +
1480 +       // setup ECC engine
1481 +       ms->ecc_eng.dev = &pdev->dev;
1482 +       ms->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED;
1483 +       ms->ecc_eng.ops = &mtk_snfi_ecc_engine_ops;
1484 +       ms->ecc_eng.priv = ms;
1485 +
1486 +       ret = nand_ecc_register_on_host_hw_engine(&ms->ecc_eng);
1487 +       if (ret) {
1488 +               dev_err(&pdev->dev, "failed to register ecc engine.\n");
1489 +               goto disable_clk;
1490 +       }
1491 +
1492 +       ctlr->num_chipselect = 1;
1493 +       ctlr->mem_ops = &mtk_snand_mem_ops;
1494 +       ctlr->mem_caps = &mtk_snand_mem_caps;
1495 +       ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
1496 +       ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD;
1497 +       ctlr->dev.of_node = pdev->dev.of_node;
1498 +       ret = spi_register_controller(ctlr);
1499 +       if (ret) {
1500 +               dev_err(&pdev->dev, "spi_register_controller failed.\n");
1501 +               goto disable_clk;
1502 +       }
1503 +
1504 +       return 0;
1505 +disable_clk:
1506 +       mtk_snand_disable_clk(ms);
1507 +release_ecc:
1508 +       mtk_ecc_release(ms->ecc);
1509 +       return ret;
1510 +}
1511 +
1512 +static int mtk_snand_remove(struct platform_device *pdev)
1513 +{
1514 +       struct spi_controller *ctlr = platform_get_drvdata(pdev);
1515 +       struct mtk_snand *ms = spi_controller_get_devdata(ctlr);
1516 +
1517 +       spi_unregister_controller(ctlr);
1518 +       mtk_snand_disable_clk(ms);
1519 +       mtk_ecc_release(ms->ecc);
1520 +       kfree(ms->buf);
1521 +       return 0;
1522 +}
1523 +
1524 +static struct platform_driver mtk_snand_driver = {
1525 +       .probe = mtk_snand_probe,
1526 +       .remove = mtk_snand_remove,
1527 +       .driver = {
1528 +               .name = "mtk-snand",
1529 +               .of_match_table = mtk_snand_ids,
1530 +       },
1531 +};
1532 +
1533 +module_platform_driver(mtk_snand_driver);
1534 +
1535 +MODULE_LICENSE("GPL");
1536 +MODULE_AUTHOR("Chuanhong Guo <gch981213@gmail.com>");
1537 +MODULE_DESCRIPTION("MeidaTek SPI-NAND Flash Controller Driver");