target/linux/ramips/patches-5.10/410-mtd-rawnand-add-driver-support-for-MT7621-nand-flash.patch

   1 From e84e2430ee0e483842b4ff013ae8a6e7e2fa2734 Mon Sep 17 00:00:00 2001
   2 From: Weijie Gao <weijie.gao@mediatek.com>
   3 Date: Wed, 1 Apr 2020 02:07:58 +0800
   4 Subject: [PATCH 1/2] mtd: rawnand: add driver support for MT7621 nand
   5  flash controller
   6
   7 This patch adds NAND flash controller driver for MediaTek MT7621 SoC.
   8
   9 The NAND flash controller is similar with controllers described in
  10 mtk_nand.c, except that the controller from MT7621 doesn't support DMA
  11 transmission, and some registers' offset and fields are different.
  12
  13 Signed-off-by: Weijie Gao <weijie.gao@mediatek.com>
  14 ---
  15  drivers/mtd/nand/raw/Kconfig       |    8 +
  16  drivers/mtd/nand/raw/Makefile      |    1 +
  17  drivers/mtd/nand/raw/mt7621_nand.c | 1348 ++++++++++++++++++++++++++++++++++++
  18  3 files changed, 1357 insertions(+)
  19  create mode 100644 drivers/mtd/nand/raw/mt7621_nand.c
  20
  21 --- a/drivers/mtd/nand/raw/Kconfig
  22 +++ b/drivers/mtd/nand/raw/Kconfig
  23 @@ -387,6 +387,14 @@ config MTD_NAND_QCOM
  24           Enables support for NAND flash chips on SoCs containing the EBI2 NAND
  25           controller. This controller is found on IPQ806x SoC.
  26
  27 +config MTD_NAND_MT7621
  28 +       tristate "MT7621 NAND controller"
  29 +       depends on SOC_MT7621 || COMPILE_TEST
  30 +       depends on HAS_IOMEM
  31 +       help
  32 +         Enables support for NAND controller on MT7621 SoC.
  33 +         This driver uses PIO mode for data transmission instead of DMA mode.
  34 +
  35  config MTD_NAND_MTK
  36         tristate "MTK NAND controller"
  37         depends on ARCH_MEDIATEK || COMPILE_TEST
  38 --- a/drivers/mtd/nand/raw/Makefile
  39 +++ b/drivers/mtd/nand/raw/Makefile
  40 @@ -51,6 +51,7 @@ obj-$(CONFIG_MTD_NAND_SUNXI)          += sunxi_n
  41  obj-$(CONFIG_MTD_NAND_HISI504)         += hisi504_nand.o
  42  obj-$(CONFIG_MTD_NAND_BRCMNAND)                += brcmnand/
  43  obj-$(CONFIG_MTD_NAND_QCOM)            += qcom_nandc.o
  44 +obj-$(CONFIG_MTD_NAND_MT7621)          += mt7621_nand.o
  45  obj-$(CONFIG_MTD_NAND_MTK)             += mtk_ecc.o mtk_nand.o
  46  obj-$(CONFIG_MTD_NAND_MXIC)            += mxic_nand.o
  47  obj-$(CONFIG_MTD_NAND_TEGRA)           += tegra_nand.o
  48 --- /dev/null
  49 +++ b/drivers/mtd/nand/raw/mt7621_nand.c
  50 @@ -0,0 +1,1353 @@
  51 +// SPDX-License-Identifier: GPL-2.0
  52 +/*
  53 + * MediaTek MT7621 NAND Flash Controller driver
  54 + *
  55 + * Copyright (C) 2020 MediaTek Inc. All Rights Reserved.
  56 + *
  57 + * Author: Weijie Gao <weijie.gao@mediatek.com>
  58 + */
  59 +
  60 +#include <linux/io.h>
  61 +#include <linux/clk.h>
  62 +#include <linux/init.h>
  63 +#include <linux/errno.h>
  64 +#include <linux/sizes.h>
  65 +#include <linux/iopoll.h>
  66 +#include <linux/kernel.h>
  67 +#include <linux/module.h>
  68 +#include <linux/mtd/mtd.h>
  69 +#include <linux/mtd/rawnand.h>
  70 +#include <linux/mtd/partitions.h>
  71 +#include <linux/platform_device.h>
  72 +#include <asm/addrspace.h>
  73 +
  74 +/* NFI core registers */
  75 +#define NFI_CNFG                       0x000
  76 +#define   CNFG_OP_MODE_S               12
  77 +#define   CNFG_OP_MODE_M               GENMASK(14, 12)
  78 +#define     CNFG_OP_CUSTOM             6
  79 +#define   CNFG_AUTO_FMT_EN             BIT(9)
  80 +#define   CNFG_HW_ECC_EN               BIT(8)
  81 +#define   CNFG_BYTE_RW                 BIT(6)
  82 +#define   CNFG_READ_MODE               BIT(1)
  83 +
  84 +#define NFI_PAGEFMT                    0x004
  85 +#define   PAGEFMT_FDM_ECC_S            12
  86 +#define   PAGEFMT_FDM_ECC_M            GENMASK(15, 12)
  87 +#define   PAGEFMT_FDM_S                        8
  88 +#define   PAGEFMT_FDM_M                        GENMASK(11, 8)
  89 +#define   PAGEFMT_SPARE_S              4
  90 +#define   PAGEFMT_SPARE_M              GENMASK(5, 4)
  91 +#define   PAGEFMT_PAGE_S               0
  92 +#define   PAGEFMT_PAGE_M               GENMASK(1, 0)
  93 +
  94 +#define NFI_CON                                0x008
  95 +#define   CON_NFI_SEC_S                        12
  96 +#define   CON_NFI_SEC_M                        GENMASK(15, 12)
  97 +#define   CON_NFI_BWR                  BIT(9)
  98 +#define   CON_NFI_BRD                  BIT(8)
  99 +#define   CON_NFI_RST                  BIT(1)
 100 +#define   CON_FIFO_FLUSH               BIT(0)
 101 +
 102 +#define NFI_ACCCON                     0x00c
 103 +#define   ACCCON_POECS_S               28
 104 +#define   ACCCON_POECS_MAX             0x0f
 105 +#define   ACCCON_POECS_DEF             3
 106 +#define   ACCCON_PRECS_S               22
 107 +#define   ACCCON_PRECS_MAX             0x3f
 108 +#define   ACCCON_PRECS_DEF             3
 109 +#define   ACCCON_C2R_S                 16
 110 +#define   ACCCON_C2R_MAX               0x3f
 111 +#define   ACCCON_C2R_DEF               7
 112 +#define   ACCCON_W2R_S                 12
 113 +#define   ACCCON_W2R_MAX               0x0f
 114 +#define   ACCCON_W2R_DEF               7
 115 +#define   ACCCON_WH_S                  8
 116 +#define   ACCCON_WH_MAX                        0x0f
 117 +#define   ACCCON_WH_DEF                        15
 118 +#define   ACCCON_WST_S                 4
 119 +#define   ACCCON_WST_MAX               0x0f
 120 +#define   ACCCON_WST_DEF               15
 121 +#define   ACCCON_WST_MIN               3
 122 +#define   ACCCON_RLT_S                 0
 123 +#define   ACCCON_RLT_MAX               0x0f
 124 +#define   ACCCON_RLT_DEF               15
 125 +#define   ACCCON_RLT_MIN               3
 126 +
 127 +#define NFI_CMD                                0x020
 128 +
 129 +#define NFI_ADDRNOB                    0x030
 130 +#define   ADDR_ROW_NOB_S               4
 131 +#define   ADDR_ROW_NOB_M               GENMASK(6, 4)
 132 +#define   ADDR_COL_NOB_S               0
 133 +#define   ADDR_COL_NOB_M               GENMASK(2, 0)
 134 +
 135 +#define NFI_COLADDR                    0x034
 136 +#define NFI_ROWADDR                    0x038
 137 +
 138 +#define NFI_STRDATA                    0x040
 139 +#define   STR_DATA                     BIT(0)
 140 +
 141 +#define NFI_CNRNB                      0x044
 142 +#define   CB2R_TIME_S                  4
 143 +#define   CB2R_TIME_M                  GENMASK(7, 4)
 144 +#define   STR_CNRNB                    BIT(0)
 145 +
 146 +#define NFI_DATAW                      0x050
 147 +#define NFI_DATAR                      0x054
 148 +
 149 +#define NFI_PIO_DIRDY                  0x058
 150 +#define   PIO_DIRDY                    BIT(0)
 151 +
 152 +#define NFI_STA                                0x060
 153 +#define   STA_NFI_FSM_S                        16
 154 +#define   STA_NFI_FSM_M                        GENMASK(19, 16)
 155 +#define     STA_FSM_CUSTOM_DATA                14
 156 +#define   STA_BUSY                     BIT(8)
 157 +#define   STA_ADDR                     BIT(1)
 158 +#define   STA_CMD                      BIT(0)
 159 +
 160 +#define NFI_ADDRCNTR                   0x070
 161 +#define   SEC_CNTR_S                   12
 162 +#define   SEC_CNTR_M                   GENMASK(15, 12)
 163 +#define   SEC_ADDR_S                   0
 164 +#define   SEC_ADDR_M                   GENMASK(9, 0)
 165 +
 166 +#define NFI_CSEL                       0x090
 167 +#define   CSEL_S                       0
 168 +#define   CSEL_M                       GENMASK(1, 0)
 169 +
 170 +#define NFI_FDM0L                      0x0a0
 171 +#define NFI_FDML(n)                    (0x0a0 + ((n) << 3))
 172 +
 173 +#define NFI_FDM0M                      0x0a4
 174 +#define NFI_FDMM(n)                    (0x0a4 + ((n) << 3))
 175 +
 176 +#define NFI_MASTER_STA                 0x210
 177 +#define   MAS_ADDR                     GENMASK(11, 9)
 178 +#define   MAS_RD                       GENMASK(8, 6)
 179 +#define   MAS_WR                       GENMASK(5, 3)
 180 +#define   MAS_RDDLY                    GENMASK(2, 0)
 181 +
 182 +/* ECC engine registers */
 183 +#define ECC_ENCCON                     0x000
 184 +#define   ENC_EN                       BIT(0)
 185 +
 186 +#define ECC_ENCCNFG                    0x004
 187 +#define   ENC_CNFG_MSG_S               16
 188 +#define   ENC_CNFG_MSG_M               GENMASK(28, 16)
 189 +#define   ENC_MODE_S                   4
 190 +#define   ENC_MODE_M                   GENMASK(5, 4)
 191 +#define     ENC_MODE_NFI               1
 192 +#define   ENC_TNUM_S                   0
 193 +#define   ENC_TNUM_M                   GENMASK(2, 0)
 194 +
 195 +#define ECC_ENCIDLE                    0x00c
 196 +#define   ENC_IDLE                     BIT(0)
 197 +
 198 +#define ECC_DECCON                     0x100
 199 +#define   DEC_EN                       BIT(0)
 200 +
 201 +#define ECC_DECCNFG                    0x104
 202 +#define   DEC_EMPTY_EN                 BIT(31)
 203 +#define   DEC_CS_S                     16
 204 +#define   DEC_CS_M                     GENMASK(28, 16)
 205 +#define   DEC_CON_S                    12
 206 +#define   DEC_CON_M                    GENMASK(13, 12)
 207 +#define     DEC_CON_EL                 2
 208 +#define   DEC_MODE_S                   4
 209 +#define   DEC_MODE_M                   GENMASK(5, 4)
 210 +#define     DEC_MODE_NFI               1
 211 +#define   DEC_TNUM_S                   0
 212 +#define   DEC_TNUM_M                   GENMASK(2, 0)
 213 +
 214 +#define ECC_DECIDLE                    0x10c
 215 +#define   DEC_IDLE                     BIT(1)
 216 +
 217 +#define ECC_DECENUM                    0x114
 218 +#define   ERRNUM_S                     2
 219 +#define   ERRNUM_M                     GENMASK(3, 0)
 220 +
 221 +#define ECC_DECDONE                    0x118
 222 +#define   DEC_DONE7                    BIT(7)
 223 +#define   DEC_DONE6                    BIT(6)
 224 +#define   DEC_DONE5                    BIT(5)
 225 +#define   DEC_DONE4                    BIT(4)
 226 +#define   DEC_DONE3                    BIT(3)
 227 +#define   DEC_DONE2                    BIT(2)
 228 +#define   DEC_DONE1                    BIT(1)
 229 +#define   DEC_DONE0                    BIT(0)
 230 +
 231 +#define ECC_DECEL(n)                   (0x11c + (n) * 4)
 232 +#define   DEC_EL_ODD_S                 16
 233 +#define   DEC_EL_EVEN_S                        0
 234 +#define   DEC_EL_M                     0x1fff
 235 +#define   DEC_EL_BYTE_POS_S            3
 236 +#define   DEC_EL_BIT_POS_M             GENMASK(2, 0)
 237 +
 238 +#define ECC_FDMADDR                    0x13c
 239 +
 240 +/* ENCIDLE and DECIDLE */
 241 +#define   ECC_IDLE                     BIT(0)
 242 +
 243 +#define ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt) \
 244 +       ((tpoecs) << ACCCON_POECS_S | (tprecs) << ACCCON_PRECS_S | \
 245 +       (tc2r) << ACCCON_C2R_S | (tw2r) << ACCCON_W2R_S | \
 246 +       (twh) << ACCCON_WH_S | (twst) << ACCCON_WST_S | (trlt))
 247 +
 248 +#define MASTER_STA_MASK                        (MAS_ADDR | MAS_RD | MAS_WR | \
 249 +                                        MAS_RDDLY)
 250 +#define NFI_RESET_TIMEOUT              1000000
 251 +#define NFI_CORE_TIMEOUT               500000
 252 +#define ECC_ENGINE_TIMEOUT             500000
 253 +
 254 +#define ECC_SECTOR_SIZE                        512
 255 +#define ECC_PARITY_BITS                        13
 256 +
 257 +#define NFI_FDM_SIZE           8
 258 +
 259 +#define MT7621_NFC_NAME                        "mt7621-nand"
 260 +
 261 +struct mt7621_nfc {
 262 +       struct nand_controller controller;
 263 +       struct nand_chip nand;
 264 +       struct clk *nfi_clk;
 265 +       struct device *dev;
 266 +
 267 +       void __iomem *nfi_regs;
 268 +       void __iomem *ecc_regs;
 269 +
 270 +       u32 spare_per_sector;
 271 +};
 272 +
 273 +static const u16 mt7621_nfi_page_size[] = { SZ_512, SZ_2K, SZ_4K };
 274 +static const u8 mt7621_nfi_spare_size[] = { 16, 26, 27, 28 };
 275 +static const u8 mt7621_ecc_strength[] = { 4, 6, 8, 10, 12 };
 276 +
 277 +static inline u32 nfi_read32(struct mt7621_nfc *nfc, u32 reg)
 278 +{
 279 +       return readl(nfc->nfi_regs + reg);
 280 +}
 281 +
 282 +static inline void nfi_write32(struct mt7621_nfc *nfc, u32 reg, u32 val)
 283 +{
 284 +       writel(val, nfc->nfi_regs + reg);
 285 +}
 286 +
 287 +static inline u16 nfi_read16(struct mt7621_nfc *nfc, u32 reg)
 288 +{
 289 +       return readw(nfc->nfi_regs + reg);
 290 +}
 291 +
 292 +static inline void nfi_write16(struct mt7621_nfc *nfc, u32 reg, u16 val)
 293 +{
 294 +       writew(val, nfc->nfi_regs + reg);
 295 +}
 296 +
 297 +static inline void ecc_write16(struct mt7621_nfc *nfc, u32 reg, u16 val)
 298 +{
 299 +       writew(val, nfc->ecc_regs + reg);
 300 +}
 301 +
 302 +static inline u32 ecc_read32(struct mt7621_nfc *nfc, u32 reg)
 303 +{
 304 +       return readl(nfc->ecc_regs + reg);
 305 +}
 306 +
 307 +static inline void ecc_write32(struct mt7621_nfc *nfc, u32 reg, u32 val)
 308 +{
 309 +       return writel(val, nfc->ecc_regs + reg);
 310 +}
 311 +
 312 +static inline u8 *oob_fdm_ptr(struct nand_chip *nand, int sect)
 313 +{
 314 +       return nand->oob_poi + sect * NFI_FDM_SIZE;
 315 +}
 316 +
 317 +static inline u8 *oob_ecc_ptr(struct mt7621_nfc *nfc, int sect)
 318 +{
 319 +       struct nand_chip *nand = &nfc->nand;
 320 +
 321 +       return nand->oob_poi + nand->ecc.steps * NFI_FDM_SIZE +
 322 +               sect * (nfc->spare_per_sector - NFI_FDM_SIZE);
 323 +}
 324 +
 325 +static inline u8 *page_data_ptr(struct nand_chip *nand, const u8 *buf,
 326 +                               int sect)
 327 +{
 328 +       return (u8 *)buf + sect * nand->ecc.size;
 329 +}
 330 +
 331 +static int mt7621_ecc_wait_idle(struct mt7621_nfc *nfc, u32 reg)
 332 +{
 333 +       struct device *dev = nfc->dev;
 334 +       u32 val;
 335 +       int ret;
 336 +
 337 +       ret = readw_poll_timeout_atomic(nfc->ecc_regs + reg, val,
 338 +                                       val & ECC_IDLE, 10,
 339 +                                       ECC_ENGINE_TIMEOUT);
 340 +       if (ret) {
 341 +               dev_warn(dev, "ECC engine timed out entering idle mode\n");
 342 +               return -EIO;
 343 +       }
 344 +
 345 +       return 0;
 346 +}
 347 +
 348 +static int mt7621_ecc_decoder_wait_done(struct mt7621_nfc *nfc, u32 sect)
 349 +{
 350 +       struct device *dev = nfc->dev;
 351 +       u32 val;
 352 +       int ret;
 353 +
 354 +       ret = readw_poll_timeout_atomic(nfc->ecc_regs + ECC_DECDONE, val,
 355 +                                       val & (1 << sect), 10,
 356 +                                       ECC_ENGINE_TIMEOUT);
 357 +
 358 +       if (ret) {
 359 +               dev_warn(dev, "ECC decoder for sector %d timed out\n",
 360 +                        sect);
 361 +               return -ETIMEDOUT;
 362 +       }
 363 +
 364 +       return 0;
 365 +}
 366 +
 367 +static void mt7621_ecc_encoder_op(struct mt7621_nfc *nfc, bool enable)
 368 +{
 369 +       mt7621_ecc_wait_idle(nfc, ECC_ENCIDLE);
 370 +       ecc_write16(nfc, ECC_ENCCON, enable ? ENC_EN : 0);
 371 +}
 372 +
 373 +static void mt7621_ecc_decoder_op(struct mt7621_nfc *nfc, bool enable)
 374 +{
 375 +       mt7621_ecc_wait_idle(nfc, ECC_DECIDLE);
 376 +       ecc_write16(nfc, ECC_DECCON, enable ? DEC_EN : 0);
 377 +}
 378 +
 379 +static int mt7621_ecc_correct_check(struct mt7621_nfc *nfc, u8 *sector_buf,
 380 +                                  u8 *fdm_buf, u32 sect)
 381 +{
 382 +       struct nand_chip *nand = &nfc->nand;
 383 +       u32 decnum, num_error_bits, fdm_end_bits;
 384 +       u32 error_locations, error_bit_loc;
 385 +       u32 error_byte_pos, error_bit_pos;
 386 +       int bitflips = 0;
 387 +       u32 i;
 388 +
 389 +       decnum = ecc_read32(nfc, ECC_DECENUM);
 390 +       num_error_bits = (decnum >> (sect << ERRNUM_S)) & ERRNUM_M;
 391 +       fdm_end_bits = (nand->ecc.size + NFI_FDM_SIZE) << 3;
 392 +
 393 +       if (!num_error_bits)
 394 +               return 0;
 395 +
 396 +       if (num_error_bits == ERRNUM_M)
 397 +               return -1;
 398 +
 399 +       for (i = 0; i < num_error_bits; i++) {
 400 +               error_locations = ecc_read32(nfc, ECC_DECEL(i / 2));
 401 +               error_bit_loc = (error_locations >> ((i % 2) * DEC_EL_ODD_S)) &
 402 +                               DEC_EL_M;
 403 +               error_byte_pos = error_bit_loc >> DEC_EL_BYTE_POS_S;
 404 +               error_bit_pos = error_bit_loc & DEC_EL_BIT_POS_M;
 405 +
 406 +               if (error_bit_loc < (nand->ecc.size << 3)) {
 407 +                       if (sector_buf) {
 408 +                               sector_buf[error_byte_pos] ^=
 409 +                                       (1 << error_bit_pos);
 410 +                       }
 411 +               } else if (error_bit_loc < fdm_end_bits) {
 412 +                       if (fdm_buf) {
 413 +                               fdm_buf[error_byte_pos - nand->ecc.size] ^=
 414 +                                       (1 << error_bit_pos);
 415 +                       }
 416 +               }
 417 +
 418 +               bitflips++;
 419 +       }
 420 +
 421 +       return bitflips;
 422 +}
 423 +
 424 +static int mt7621_nfc_wait_write_completion(struct mt7621_nfc *nfc,
 425 +                                           struct nand_chip *nand)
 426 +{
 427 +       struct device *dev = nfc->dev;
 428 +       u16 val;
 429 +       int ret;
 430 +
 431 +       ret = readw_poll_timeout_atomic(nfc->nfi_regs + NFI_ADDRCNTR, val,
 432 +               ((val & SEC_CNTR_M) >> SEC_CNTR_S) >= nand->ecc.steps, 10,
 433 +               NFI_CORE_TIMEOUT);
 434 +
 435 +       if (ret) {
 436 +               dev_warn(dev, "NFI core write operation timed out\n");
 437 +               return -ETIMEDOUT;
 438 +       }
 439 +
 440 +       return ret;
 441 +}
 442 +
 443 +static void mt7621_nfc_hw_reset(struct mt7621_nfc *nfc)
 444 +{
 445 +       u32 val;
 446 +       int ret;
 447 +
 448 +       /* reset all registers and force the NFI master to terminate */
 449 +       nfi_write16(nfc, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST);
 450 +
 451 +       /* wait for the master to finish the last transaction */
 452 +       ret = readw_poll_timeout(nfc->nfi_regs + NFI_MASTER_STA, val,
 453 +                                !(val & MASTER_STA_MASK), 50,
 454 +                                NFI_RESET_TIMEOUT);
 455 +       if (ret) {
 456 +               dev_warn(nfc->dev, "Failed to reset NFI master in %dms\n",
 457 +                        NFI_RESET_TIMEOUT);
 458 +       }
 459 +
 460 +       /* ensure any status register affected by the NFI master is reset */
 461 +       nfi_write16(nfc, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST);
 462 +       nfi_write16(nfc, NFI_STRDATA, 0);
 463 +}
 464 +
 465 +static inline void mt7621_nfc_hw_init(struct mt7621_nfc *nfc)
 466 +{
 467 +       u32 acccon;
 468 +
 469 +       /*
 470 +        * CNRNB: nand ready/busy register
 471 +        * -------------------------------
 472 +        * 7:4: timeout register for polling the NAND busy/ready signal
 473 +        * 0  : poll the status of the busy/ready signal after [7:4]*16 cycles.
 474 +        */
 475 +       nfi_write16(nfc, NFI_CNRNB, CB2R_TIME_M | STR_CNRNB);
 476 +
 477 +       mt7621_nfc_hw_reset(nfc);
 478 +
 479 +       /* Apply default access timing */
 480 +       acccon = ACCTIMING(ACCCON_POECS_DEF, ACCCON_PRECS_DEF, ACCCON_C2R_DEF,
 481 +                          ACCCON_W2R_DEF, ACCCON_WH_DEF, ACCCON_WST_DEF,
 482 +                          ACCCON_RLT_DEF);
 483 +
 484 +       nfi_write32(nfc, NFI_ACCCON, acccon);
 485 +}
 486 +
 487 +static int mt7621_nfc_send_command(struct mt7621_nfc *nfc, u8 command)
 488 +{
 489 +       struct device *dev = nfc->dev;
 490 +       u32 val;
 491 +       int ret;
 492 +
 493 +       nfi_write32(nfc, NFI_CMD, command);
 494 +
 495 +       ret = readl_poll_timeout_atomic(nfc->nfi_regs + NFI_STA, val,
 496 +                                       !(val & STA_CMD), 10,
 497 +                                       NFI_CORE_TIMEOUT);
 498 +       if (ret) {
 499 +               dev_warn(dev, "NFI core timed out entering command mode\n");
 500 +               return -EIO;
 501 +       }
 502 +
 503 +       return 0;
 504 +}
 505 +
 506 +static int mt7621_nfc_send_address_byte(struct mt7621_nfc *nfc, int addr)
 507 +{
 508 +       struct device *dev = nfc->dev;
 509 +       u32 val;
 510 +       int ret;
 511 +
 512 +       nfi_write32(nfc, NFI_COLADDR, addr);
 513 +       nfi_write32(nfc, NFI_ROWADDR, 0);
 514 +       nfi_write16(nfc, NFI_ADDRNOB, 1);
 515 +
 516 +       ret = readl_poll_timeout_atomic(nfc->nfi_regs + NFI_STA, val,
 517 +                                       !(val & STA_ADDR), 10,
 518 +                                       NFI_CORE_TIMEOUT);
 519 +       if (ret) {
 520 +               dev_warn(dev, "NFI core timed out entering address mode\n");
 521 +               return -EIO;
 522 +       }
 523 +
 524 +       return 0;
 525 +}
 526 +
 527 +static int mt7621_nfc_send_address(struct mt7621_nfc *nfc, const u8 *addr,
 528 +                                  unsigned int naddrs)
 529 +{
 530 +       int ret;
 531 +
 532 +       while (naddrs) {
 533 +               ret = mt7621_nfc_send_address_byte(nfc, *addr);
 534 +               if (ret)
 535 +                       return ret;
 536 +
 537 +               addr++;
 538 +               naddrs--;
 539 +       }
 540 +
 541 +       return 0;
 542 +}
 543 +
 544 +static void mt7621_nfc_wait_pio_ready(struct mt7621_nfc *nfc)
 545 +{
 546 +       struct device *dev = nfc->dev;
 547 +       int ret;
 548 +       u16 val;
 549 +
 550 +       ret = readw_poll_timeout_atomic(nfc->nfi_regs + NFI_PIO_DIRDY, val,
 551 +                                       val & PIO_DIRDY, 10,
 552 +                                       NFI_CORE_TIMEOUT);
 553 +       if (ret < 0)
 554 +               dev_err(dev, "NFI core PIO mode not ready\n");
 555 +}
 556 +
 557 +static u32 mt7621_nfc_pio_read(struct mt7621_nfc *nfc, bool br)
 558 +{
 559 +       u32 reg;
 560 +
 561 +       /* after each byte read, the NFI_STA reg is reset by the hardware */
 562 +       reg = (nfi_read32(nfc, NFI_STA) & STA_NFI_FSM_M) >> STA_NFI_FSM_S;
 563 +       if (reg != STA_FSM_CUSTOM_DATA) {
 564 +               reg = nfi_read16(nfc, NFI_CNFG);
 565 +               reg |= CNFG_READ_MODE | CNFG_BYTE_RW;
 566 +               if (!br)
 567 +                       reg &= ~CNFG_BYTE_RW;
 568 +               nfi_write16(nfc, NFI_CNFG, reg);
 569 +
 570 +               /*
 571 +                * set to max sector to allow the HW to continue reading over
 572 +                * unaligned accesses
 573 +                */
 574 +               nfi_write16(nfc, NFI_CON, CON_NFI_SEC_M | CON_NFI_BRD);
 575 +
 576 +               /* trigger to fetch data */
 577 +               nfi_write16(nfc, NFI_STRDATA, STR_DATA);
 578 +       }
 579 +
 580 +       mt7621_nfc_wait_pio_ready(nfc);
 581 +
 582 +       return nfi_read32(nfc, NFI_DATAR);
 583 +}
 584 +
 585 +static void mt7621_nfc_read_data(struct mt7621_nfc *nfc, u8 *buf, u32 len)
 586 +{
 587 +       while (((uintptr_t)buf & 3) && len) {
 588 +               *buf = mt7621_nfc_pio_read(nfc, true);
 589 +               buf++;
 590 +               len--;
 591 +       }
 592 +
 593 +       while (len >= 4) {
 594 +               *(u32 *)buf = mt7621_nfc_pio_read(nfc, false);
 595 +               buf += 4;
 596 +               len -= 4;
 597 +       }
 598 +
 599 +       while (len) {
 600 +               *buf = mt7621_nfc_pio_read(nfc, true);
 601 +               buf++;
 602 +               len--;
 603 +       }
 604 +}
 605 +
 606 +static void mt7621_nfc_read_data_discard(struct mt7621_nfc *nfc, u32 len)
 607 +{
 608 +       while (len >= 4) {
 609 +               mt7621_nfc_pio_read(nfc, false);
 610 +               len -= 4;
 611 +       }
 612 +
 613 +       while (len) {
 614 +               mt7621_nfc_pio_read(nfc, true);
 615 +               len--;
 616 +       }
 617 +}
 618 +
 619 +static void mt7621_nfc_pio_write(struct mt7621_nfc *nfc, u32 val, bool bw)
 620 +{
 621 +       u32 reg;
 622 +
 623 +       reg = (nfi_read32(nfc, NFI_STA) & STA_NFI_FSM_M) >> STA_NFI_FSM_S;
 624 +       if (reg != STA_FSM_CUSTOM_DATA) {
 625 +               reg = nfi_read16(nfc, NFI_CNFG);
 626 +               reg &= ~(CNFG_READ_MODE | CNFG_BYTE_RW);
 627 +               if (bw)
 628 +                       reg |= CNFG_BYTE_RW;
 629 +               nfi_write16(nfc, NFI_CNFG, reg);
 630 +
 631 +               nfi_write16(nfc, NFI_CON, CON_NFI_SEC_M | CON_NFI_BWR);
 632 +               nfi_write16(nfc, NFI_STRDATA, STR_DATA);
 633 +       }
 634 +
 635 +       mt7621_nfc_wait_pio_ready(nfc);
 636 +       nfi_write32(nfc, NFI_DATAW, val);
 637 +}
 638 +
 639 +static void mt7621_nfc_write_data(struct mt7621_nfc *nfc, const u8 *buf,
 640 +                                 u32 len)
 641 +{
 642 +       while (((uintptr_t)buf & 3) && len) {
 643 +               mt7621_nfc_pio_write(nfc, *buf, true);
 644 +               buf++;
 645 +               len--;
 646 +       }
 647 +
 648 +       while (len >= 4) {
 649 +               mt7621_nfc_pio_write(nfc, *(const u32 *)buf, false);
 650 +               buf += 4;
 651 +               len -= 4;
 652 +       }
 653 +
 654 +       while (len) {
 655 +               mt7621_nfc_pio_write(nfc, *buf, true);
 656 +               buf++;
 657 +               len--;
 658 +       }
 659 +}
 660 +
 661 +static void mt7621_nfc_write_data_empty(struct mt7621_nfc *nfc, u32 len)
 662 +{
 663 +       while (len >= 4) {
 664 +               mt7621_nfc_pio_write(nfc, 0xffffffff, false);
 665 +               len -= 4;
 666 +       }
 667 +
 668 +       while (len) {
 669 +               mt7621_nfc_pio_write(nfc, 0xff, true);
 670 +               len--;
 671 +       }
 672 +}
 673 +
 674 +static int mt7621_nfc_dev_ready(struct mt7621_nfc *nfc,
 675 +                               unsigned int timeout_ms)
 676 +{
 677 +       u32 val;
 678 +
 679 +       return readl_poll_timeout_atomic(nfc->nfi_regs + NFI_STA, val,
 680 +                                        !(val & STA_BUSY), 10,
 681 +                                        timeout_ms * 1000);
 682 +}
 683 +
 684 +static int mt7621_nfc_exec_instr(struct nand_chip *nand,
 685 +                                const struct nand_op_instr *instr)
 686 +{
 687 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
 688 +
 689 +       switch (instr->type) {
 690 +       case NAND_OP_CMD_INSTR:
 691 +               mt7621_nfc_hw_reset(nfc);
 692 +               nfi_write16(nfc, NFI_CNFG, CNFG_OP_CUSTOM << CNFG_OP_MODE_S);
 693 +               return mt7621_nfc_send_command(nfc, instr->ctx.cmd.opcode);
 694 +       case NAND_OP_ADDR_INSTR:
 695 +               return mt7621_nfc_send_address(nfc, instr->ctx.addr.addrs,
 696 +                                              instr->ctx.addr.naddrs);
 697 +       case NAND_OP_DATA_IN_INSTR:
 698 +               mt7621_nfc_read_data(nfc, instr->ctx.data.buf.in,
 699 +                                    instr->ctx.data.len);
 700 +               return 0;
 701 +       case NAND_OP_DATA_OUT_INSTR:
 702 +               mt7621_nfc_write_data(nfc, instr->ctx.data.buf.out,
 703 +                                     instr->ctx.data.len);
 704 +               return 0;
 705 +       case NAND_OP_WAITRDY_INSTR:
 706 +               return mt7621_nfc_dev_ready(nfc,
 707 +                                           instr->ctx.waitrdy.timeout_ms);
 708 +       default:
 709 +               WARN_ONCE(1, "unsupported NAND instruction type: %d\n",
 710 +                         instr->type);
 711 +
 712 +               return -EINVAL;
 713 +       }
 714 +}
 715 +
 716 +static int mt7621_nfc_exec_op(struct nand_chip *nand,
 717 +                             const struct nand_operation *op, bool check_only)
 718 +{
 719 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
 720 +       int i, ret;
 721 +
 722 +       if (check_only)
 723 +               return 0;
 724 +
 725 +       /* Only CS0 available */
 726 +       nfi_write16(nfc, NFI_CSEL, 0);
 727 +
 728 +       for (i = 0; i < op->ninstrs; i++) {
 729 +               ret = mt7621_nfc_exec_instr(nand, &op->instrs[i]);
 730 +               if (ret)
 731 +                       return ret;
 732 +       }
 733 +
 734 +       return 0;
 735 +}
 736 +
 737 +static int mt7621_nfc_setup_interface(struct nand_chip *nand, int csline,
 738 +                                     const struct nand_interface_config *conf)
 739 +{
 740 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
 741 +       const struct nand_sdr_timings *timings;
 742 +       u32 acccon, temp, rate, tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt;
 743 +
 744 +       if (!nfc->nfi_clk)
 745 +               return -ENOTSUPP;
 746 +
 747 +       timings = nand_get_sdr_timings(conf);
 748 +       if (IS_ERR(timings))
 749 +               return -ENOTSUPP;
 750 +
 751 +       rate = clk_get_rate(nfc->nfi_clk);
 752 +
 753 +       /* turn clock rate into KHZ */
 754 +       rate /= 1000;
 755 +
 756 +       tpoecs = max(timings->tALH_min, timings->tCLH_min) / 1000;
 757 +       tpoecs = DIV_ROUND_UP(tpoecs * rate, 1000000);
 758 +       tpoecs = min_t(u32, tpoecs, ACCCON_POECS_MAX);
 759 +
 760 +       tprecs = max(timings->tCLS_min, timings->tALS_min) / 1000;
 761 +       tprecs = DIV_ROUND_UP(tprecs * rate, 1000000);
 762 +       tprecs = min_t(u32, tprecs, ACCCON_PRECS_MAX);
 763 +
 764 +       /* sdr interface has no tCR which means CE# low to RE# low */
 765 +       tc2r = 0;
 766 +
 767 +       tw2r = timings->tWHR_min / 1000;
 768 +       tw2r = DIV_ROUND_UP(tw2r * rate, 1000000);
 769 +       tw2r = DIV_ROUND_UP(tw2r - 1, 2);
 770 +       tw2r = min_t(u32, tw2r, ACCCON_W2R_MAX);
 771 +
 772 +       twh = max(timings->tREH_min, timings->tWH_min) / 1000;
 773 +       twh = DIV_ROUND_UP(twh * rate, 1000000) - 1;
 774 +       twh = min_t(u32, twh, ACCCON_WH_MAX);
 775 +
 776 +       /* Calculate real WE#/RE# hold time in nanosecond */
 777 +       temp = (twh + 1) * 1000000 / rate;
 778 +       /* nanosecond to picosecond */
 779 +       temp *= 1000;
 780 +
 781 +       /*
 782 +        * WE# low level time should be expaned to meet WE# pulse time
 783 +        * and WE# cycle time at the same time.
 784 +        */
 785 +       if (temp < timings->tWC_min)
 786 +               twst = timings->tWC_min - temp;
 787 +       else
 788 +               twst = 0;
 789 +       twst = max(timings->tWP_min, twst) / 1000;
 790 +       twst = DIV_ROUND_UP(twst * rate, 1000000) - 1;
 791 +       twst = min_t(u32, twst, ACCCON_WST_MAX);
 792 +
 793 +       /*
 794 +        * RE# low level time should be expaned to meet RE# pulse time
 795 +        * and RE# cycle time at the same time.
 796 +        */
 797 +       if (temp < timings->tRC_min)
 798 +               trlt = timings->tRC_min - temp;
 799 +       else
 800 +               trlt = 0;
 801 +       trlt = max(trlt, timings->tRP_min) / 1000;
 802 +       trlt = DIV_ROUND_UP(trlt * rate, 1000000) - 1;
 803 +       trlt = min_t(u32, trlt, ACCCON_RLT_MAX);
 804 +
 805 +       if (csline == NAND_DATA_IFACE_CHECK_ONLY) {
 806 +               if (twst < ACCCON_WST_MIN || trlt < ACCCON_RLT_MIN)
 807 +                       return -ENOTSUPP;
 808 +       }
 809 +
 810 +       acccon = ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt);
 811 +
 812 +       dev_info(nfc->dev, "Using programmed access timing: %08x\n", acccon);
 813 +
 814 +       nfi_write32(nfc, NFI_ACCCON, acccon);
 815 +
 816 +       return 0;
 817 +}
 818 +
 819 +static int mt7621_nfc_calc_ecc_strength(struct mt7621_nfc *nfc,
 820 +                                       u32 avail_ecc_bytes)
 821 +{
 822 +       struct nand_chip *nand = &nfc->nand;
 823 +       struct mtd_info *mtd = nand_to_mtd(nand);
 824 +       u32 strength;
 825 +       int i;
 826 +
 827 +       strength = avail_ecc_bytes * 8 / ECC_PARITY_BITS;
 828 +
 829 +       /* Find the closest supported ecc strength */
 830 +       for (i = ARRAY_SIZE(mt7621_ecc_strength) - 1; i >= 0; i--) {
 831 +               if (mt7621_ecc_strength[i] <= strength)
 832 +                       break;
 833 +       }
 834 +
 835 +       if (unlikely(i < 0)) {
 836 +               dev_err(nfc->dev, "OOB size (%u) is not supported\n",
 837 +                       mtd->oobsize);
 838 +               return -EINVAL;
 839 +       }
 840 +
 841 +       nand->ecc.strength = mt7621_ecc_strength[i];
 842 +       nand->ecc.bytes =
 843 +               DIV_ROUND_UP(nand->ecc.strength * ECC_PARITY_BITS, 8);
 844 +
 845 +       dev_info(nfc->dev, "ECC strength adjusted to %u bits\n",
 846 +                nand->ecc.strength);
 847 +
 848 +       return i;
 849 +}
 850 +
 851 +static int mt7621_nfc_set_spare_per_sector(struct mt7621_nfc *nfc)
 852 +{
 853 +       struct nand_chip *nand = &nfc->nand;
 854 +       struct mtd_info *mtd = nand_to_mtd(nand);
 855 +       u32 size;
 856 +       int i;
 857 +
 858 +       size = nand->ecc.bytes + NFI_FDM_SIZE;
 859 +
 860 +       /* Find the closest supported spare size */
 861 +       for (i = 0; i < ARRAY_SIZE(mt7621_nfi_spare_size); i++) {
 862 +               if (mt7621_nfi_spare_size[i] >= size)
 863 +                       break;
 864 +       }
 865 +
 866 +       if (unlikely(i >= ARRAY_SIZE(mt7621_nfi_spare_size))) {
 867 +               dev_err(nfc->dev, "OOB size (%u) is not supported\n",
 868 +                       mtd->oobsize);
 869 +               return -EINVAL;
 870 +       }
 871 +
 872 +       nfc->spare_per_sector = mt7621_nfi_spare_size[i];
 873 +
 874 +       return i;
 875 +}
 876 +
 877 +static int mt7621_nfc_ecc_init(struct mt7621_nfc *nfc)
 878 +{
 879 +       struct nand_chip *nand = &nfc->nand;
 880 +       struct mtd_info *mtd = nand_to_mtd(nand);
 881 +       u32 spare_per_sector, encode_block_size, decode_block_size;
 882 +       u32 ecc_enccfg, ecc_deccfg;
 883 +       int ecc_cap;
 884 +
 885 +       /* Only hardware ECC mode is supported */
 886 +       if (nand->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) {
 887 +               dev_err(nfc->dev, "Only hardware ECC mode is supported\n");
 888 +               return -EINVAL;
 889 +       }
 890 +
 891 +       nand->ecc.size = ECC_SECTOR_SIZE;
 892 +       nand->ecc.steps = mtd->writesize / nand->ecc.size;
 893 +
 894 +       spare_per_sector = mtd->oobsize / nand->ecc.steps;
 895 +
 896 +       ecc_cap = mt7621_nfc_calc_ecc_strength(nfc,
 897 +               spare_per_sector - NFI_FDM_SIZE);
 898 +       if (ecc_cap < 0)
 899 +               return ecc_cap;
 900 +
 901 +       /* Sector + FDM */
 902 +       encode_block_size = (nand->ecc.size + NFI_FDM_SIZE) * 8;
 903 +       ecc_enccfg = ecc_cap | (ENC_MODE_NFI << ENC_MODE_S) |
 904 +                    (encode_block_size << ENC_CNFG_MSG_S);
 905 +
 906 +       /* Sector + FDM + ECC parity bits */
 907 +       decode_block_size = ((nand->ecc.size + NFI_FDM_SIZE) * 8) +
 908 +                           nand->ecc.strength * ECC_PARITY_BITS;
 909 +       ecc_deccfg = ecc_cap | (DEC_MODE_NFI << DEC_MODE_S) |
 910 +                    (decode_block_size << DEC_CS_S) |
 911 +                    (DEC_CON_EL << DEC_CON_S) | DEC_EMPTY_EN;
 912 +
 913 +       mt7621_ecc_encoder_op(nfc, false);
 914 +       ecc_write32(nfc, ECC_ENCCNFG, ecc_enccfg);
 915 +
 916 +       mt7621_ecc_decoder_op(nfc, false);
 917 +       ecc_write32(nfc, ECC_DECCNFG, ecc_deccfg);
 918 +
 919 +       return 0;
 920 +}
 921 +
 922 +static int mt7621_nfc_set_page_format(struct mt7621_nfc *nfc)
 923 +{
 924 +       struct nand_chip *nand = &nfc->nand;
 925 +       struct mtd_info *mtd = nand_to_mtd(nand);
 926 +       int i, spare_size;
 927 +       u32 pagefmt;
 928 +
 929 +       spare_size = mt7621_nfc_set_spare_per_sector(nfc);
 930 +       if (spare_size < 0)
 931 +               return spare_size;
 932 +
 933 +       for (i = 0; i < ARRAY_SIZE(mt7621_nfi_page_size); i++) {
 934 +               if (mt7621_nfi_page_size[i] == mtd->writesize)
 935 +                       break;
 936 +       }
 937 +
 938 +       if (unlikely(i >= ARRAY_SIZE(mt7621_nfi_page_size))) {
 939 +               dev_err(nfc->dev, "Page size (%u) is not supported\n",
 940 +                       mtd->writesize);
 941 +               return -EINVAL;
 942 +       }
 943 +
 944 +       pagefmt = i | (spare_size << PAGEFMT_SPARE_S) |
 945 +                 (NFI_FDM_SIZE << PAGEFMT_FDM_S) |
 946 +                 (NFI_FDM_SIZE << PAGEFMT_FDM_ECC_S);
 947 +
 948 +       nfi_write16(nfc, NFI_PAGEFMT, pagefmt);
 949 +
 950 +       return 0;
 951 +}
 952 +
 953 +static int mt7621_nfc_attach_chip(struct nand_chip *nand)
 954 +{
 955 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
 956 +       int ret;
 957 +
 958 +       if (nand->options & NAND_BUSWIDTH_16) {
 959 +               dev_err(nfc->dev, "16-bit buswidth is not supported");
 960 +               return -EINVAL;
 961 +       }
 962 +
 963 +       ret = mt7621_nfc_ecc_init(nfc);
 964 +       if (ret)
 965 +               return ret;
 966 +
 967 +       return mt7621_nfc_set_page_format(nfc);
 968 +}
 969 +
 970 +static const struct nand_controller_ops mt7621_nfc_controller_ops = {
 971 +       .attach_chip = mt7621_nfc_attach_chip,
 972 +       .exec_op = mt7621_nfc_exec_op,
 973 +       .setup_interface = mt7621_nfc_setup_interface,
 974 +};
 975 +
 976 +static int mt7621_nfc_ooblayout_free(struct mtd_info *mtd, int section,
 977 +                                    struct mtd_oob_region *oob_region)
 978 +{
 979 +       struct nand_chip *nand = mtd_to_nand(mtd);
 980 +
 981 +       if (section >= nand->ecc.steps)
 982 +               return -ERANGE;
 983 +
 984 +       oob_region->length = NFI_FDM_SIZE - 1;
 985 +       oob_region->offset = section * NFI_FDM_SIZE + 1;
 986 +
 987 +       return 0;
 988 +}
 989 +
 990 +static int mt7621_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
 991 +                                   struct mtd_oob_region *oob_region)
 992 +{
 993 +       struct nand_chip *nand = mtd_to_nand(mtd);
 994 +
 995 +       if (section)
 996 +               return -ERANGE;
 997 +
 998 +       oob_region->offset = NFI_FDM_SIZE * nand->ecc.steps;
 999 +       oob_region->length = mtd->oobsize - oob_region->offset;
1000 +
1001 +       return 0;
1002 +}
1003 +
1004 +static const struct mtd_ooblayout_ops mt7621_nfc_ooblayout_ops = {
1005 +       .free = mt7621_nfc_ooblayout_free,
1006 +       .ecc = mt7621_nfc_ooblayout_ecc,
1007 +};
1008 +
1009 +static void mt7621_nfc_write_fdm(struct mt7621_nfc *nfc)
1010 +{
1011 +       struct nand_chip *nand = &nfc->nand;
1012 +       u32 vall, valm;
1013 +       u8 *oobptr;
1014 +       int i, j;
1015 +
1016 +       for (i = 0; i < nand->ecc.steps; i++) {
1017 +               vall = 0;
1018 +               valm = 0;
1019 +               oobptr = oob_fdm_ptr(nand, i);
1020 +
1021 +               for (j = 0; j < 4; j++)
1022 +                       vall |= (u32)oobptr[j] << (j * 8);
1023 +
1024 +               for (j = 0; j < 4; j++)
1025 +                       valm |= (u32)oobptr[j + 4] << (j * 8);
1026 +
1027 +               nfi_write32(nfc, NFI_FDML(i), vall);
1028 +               nfi_write32(nfc, NFI_FDMM(i), valm);
1029 +       }
1030 +}
1031 +
1032 +static void mt7621_nfc_read_sector_fdm(struct mt7621_nfc *nfc, u32 sect)
1033 +{
1034 +       struct nand_chip *nand = &nfc->nand;
1035 +       u32 vall, valm;
1036 +       u8 *oobptr;
1037 +       int i;
1038 +
1039 +       vall = nfi_read32(nfc, NFI_FDML(sect));
1040 +       valm = nfi_read32(nfc, NFI_FDMM(sect));
1041 +       oobptr = oob_fdm_ptr(nand, sect);
1042 +
1043 +       for (i = 0; i < 4; i++)
1044 +               oobptr[i] = (vall >> (i * 8)) & 0xff;
1045 +
1046 +       for (i = 0; i < 4; i++)
1047 +               oobptr[i + 4] = (valm >> (i * 8)) & 0xff;
1048 +}
1049 +
1050 +static int mt7621_nfc_read_page_hwecc(struct nand_chip *nand, uint8_t *buf,
1051 +                                     int oob_required, int page)
1052 +{
1053 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
1054 +       struct mtd_info *mtd = nand_to_mtd(nand);
1055 +       int bitflips = 0;
1056 +       int rc, i;
1057 +
1058 +       nand_read_page_op(nand, page, 0, NULL, 0);
1059 +
1060 +       nfi_write16(nfc, NFI_CNFG, (CNFG_OP_CUSTOM << CNFG_OP_MODE_S) |
1061 +                   CNFG_READ_MODE | CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
1062 +
1063 +       mt7621_ecc_decoder_op(nfc, true);
1064 +
1065 +       nfi_write16(nfc, NFI_CON,
1066 +                   CON_NFI_BRD | (nand->ecc.steps << CON_NFI_SEC_S));
1067 +
1068 +       for (i = 0; i < nand->ecc.steps; i++) {
1069 +               if (buf)
1070 +                       mt7621_nfc_read_data(nfc, page_data_ptr(nand, buf, i),
1071 +                                            nand->ecc.size);
1072 +               else
1073 +                       mt7621_nfc_read_data_discard(nfc, nand->ecc.size);
1074 +
1075 +               rc = mt7621_ecc_decoder_wait_done(nfc, i);
1076 +
1077 +               mt7621_nfc_read_sector_fdm(nfc, i);
1078 +
1079 +               if (rc < 0) {
1080 +                       bitflips = -EIO;
1081 +                       continue;
1082 +               }
1083 +
1084 +               rc = mt7621_ecc_correct_check(nfc,
1085 +                       buf ? page_data_ptr(nand, buf, i) : NULL,
1086 +                       oob_fdm_ptr(nand, i), i);
1087 +
1088 +               if (rc < 0) {
1089 +                       dev_warn(nfc->dev,
1090 +                                "Uncorrectable ECC error at page %d.%d\n",
1091 +                                page, i);
1092 +                       bitflips = -EBADMSG;
1093 +                       mtd->ecc_stats.failed++;
1094 +               } else if (bitflips >= 0) {
1095 +                       bitflips += rc;
1096 +                       mtd->ecc_stats.corrected += rc;
1097 +               }
1098 +       }
1099 +
1100 +       mt7621_ecc_decoder_op(nfc, false);
1101 +
1102 +       nfi_write16(nfc, NFI_CON, 0);
1103 +
1104 +       return bitflips;
1105 +}
1106 +
1107 +static int mt7621_nfc_read_page_raw(struct nand_chip *nand, uint8_t *buf,
1108 +                                   int oob_required, int page)
1109 +{
1110 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
1111 +       int i;
1112 +
1113 +       nand_read_page_op(nand, page, 0, NULL, 0);
1114 +
1115 +       nfi_write16(nfc, NFI_CNFG, (CNFG_OP_CUSTOM << CNFG_OP_MODE_S) |
1116 +                   CNFG_READ_MODE);
1117 +
1118 +       nfi_write16(nfc, NFI_CON,
1119 +                   CON_NFI_BRD | (nand->ecc.steps << CON_NFI_SEC_S));
1120 +
1121 +       for (i = 0; i < nand->ecc.steps; i++) {
1122 +               /* Read data */
1123 +               if (buf)
1124 +                       mt7621_nfc_read_data(nfc, page_data_ptr(nand, buf, i),
1125 +                                            nand->ecc.size);
1126 +               else
1127 +                       mt7621_nfc_read_data_discard(nfc, nand->ecc.size);
1128 +
1129 +               /* Read FDM */
1130 +               mt7621_nfc_read_data(nfc, oob_fdm_ptr(nand, i), NFI_FDM_SIZE);
1131 +
1132 +               /* Read ECC parity data */
1133 +               mt7621_nfc_read_data(nfc, oob_ecc_ptr(nfc, i),
1134 +                                    nfc->spare_per_sector - NFI_FDM_SIZE);
1135 +       }
1136 +
1137 +       nfi_write16(nfc, NFI_CON, 0);
1138 +
1139 +       return 0;
1140 +}
1141 +
1142 +static int mt7621_nfc_read_oob_hwecc(struct nand_chip *nand, int page)
1143 +{
1144 +       return mt7621_nfc_read_page_hwecc(nand, NULL, 1, page);
1145 +}
1146 +
1147 +static int mt7621_nfc_read_oob_raw(struct nand_chip *nand, int page)
1148 +{
1149 +       return mt7621_nfc_read_page_raw(nand, NULL, 1, page);
1150 +}
1151 +
1152 +static int mt7621_nfc_check_empty_page(struct nand_chip *nand, const u8 *buf)
1153 +{
1154 +       struct mtd_info *mtd = nand_to_mtd(nand);
1155 +       uint32_t i, j;
1156 +       u8 *oobptr;
1157 +
1158 +       if (buf) {
1159 +               for (i = 0; i < mtd->writesize; i++)
1160 +                       if (buf[i] != 0xff)
1161 +                               return 0;
1162 +       }
1163 +
1164 +       for (i = 0; i < nand->ecc.steps; i++) {
1165 +               oobptr = oob_fdm_ptr(nand, i);
1166 +               for (j = 0; j < NFI_FDM_SIZE; j++)
1167 +                       if (oobptr[j] != 0xff)
1168 +                               return 0;
1169 +       }
1170 +
1171 +       return 1;
1172 +}
1173 +
1174 +static int mt7621_nfc_write_page_hwecc(struct nand_chip *nand,
1175 +                                      const uint8_t *buf, int oob_required,
1176 +                                      int page)
1177 +{
1178 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
1179 +       struct mtd_info *mtd = nand_to_mtd(nand);
1180 +
1181 +       if (mt7621_nfc_check_empty_page(nand, buf)) {
1182 +               /*
1183 +                * MT7621 ECC engine always generates parity code for input
1184 +                * pages, even for empty pages. Doing so will write back ECC
1185 +                * parity code to the oob region, which means such pages will
1186 +                * no longer be empty pages.
1187 +                *
1188 +                * To avoid this, stop write operation if current page is an
1189 +                * empty page.
1190 +                */
1191 +               return 0;
1192 +       }
1193 +
1194 +       nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1195 +
1196 +       nfi_write16(nfc, NFI_CNFG, (CNFG_OP_CUSTOM << CNFG_OP_MODE_S) |
1197 +                  CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
1198 +
1199 +       mt7621_ecc_encoder_op(nfc, true);
1200 +
1201 +       mt7621_nfc_write_fdm(nfc);
1202 +
1203 +       nfi_write16(nfc, NFI_CON,
1204 +                   CON_NFI_BWR | (nand->ecc.steps << CON_NFI_SEC_S));
1205 +
1206 +       if (buf)
1207 +               mt7621_nfc_write_data(nfc, buf, mtd->writesize);
1208 +       else
1209 +               mt7621_nfc_write_data_empty(nfc, mtd->writesize);
1210 +
1211 +       mt7621_nfc_wait_write_completion(nfc, nand);
1212 +
1213 +       mt7621_ecc_encoder_op(nfc, false);
1214 +
1215 +       nfi_write16(nfc, NFI_CON, 0);
1216 +
1217 +       return nand_prog_page_end_op(nand);
1218 +}
1219 +
1220 +static int mt7621_nfc_write_page_raw(struct nand_chip *nand,
1221 +                                    const uint8_t *buf, int oob_required,
1222 +                                    int page)
1223 +{
1224 +       struct mt7621_nfc *nfc = nand_get_controller_data(nand);
1225 +       int i;
1226 +
1227 +       nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1228 +
1229 +       nfi_write16(nfc, NFI_CNFG, (CNFG_OP_CUSTOM << CNFG_OP_MODE_S));
1230 +
1231 +       nfi_write16(nfc, NFI_CON,
1232 +                   CON_NFI_BWR | (nand->ecc.steps << CON_NFI_SEC_S));
1233 +
1234 +       for (i = 0; i < nand->ecc.steps; i++) {
1235 +               /* Write data */
1236 +               if (buf)
1237 +                       mt7621_nfc_write_data(nfc, page_data_ptr(nand, buf, i),
1238 +                                             nand->ecc.size);
1239 +               else
1240 +                       mt7621_nfc_write_data_empty(nfc, nand->ecc.size);
1241 +
1242 +               /* Write FDM */
1243 +               mt7621_nfc_write_data(nfc, oob_fdm_ptr(nand, i),
1244 +                                     NFI_FDM_SIZE);
1245 +
1246 +               /* Write dummy ECC parity data */
1247 +               mt7621_nfc_write_data_empty(nfc, nfc->spare_per_sector -
1248 +                                           NFI_FDM_SIZE);
1249 +       }
1250 +
1251 +       mt7621_nfc_wait_write_completion(nfc, nand);
1252 +
1253 +       nfi_write16(nfc, NFI_CON, 0);
1254 +
1255 +       return nand_prog_page_end_op(nand);
1256 +}
1257 +
1258 +static int mt7621_nfc_write_oob_hwecc(struct nand_chip *nand, int page)
1259 +{
1260 +       return mt7621_nfc_write_page_hwecc(nand, NULL, 1, page);
1261 +}
1262 +
1263 +static int mt7621_nfc_write_oob_raw(struct nand_chip *nand, int page)
1264 +{
1265 +       return mt7621_nfc_write_page_raw(nand, NULL, 1, page);
1266 +}
1267 +
1268 +static int mt7621_nfc_init_chip(struct mt7621_nfc *nfc)
1269 +{
1270 +       struct nand_chip *nand = &nfc->nand;
1271 +       struct mtd_info *mtd;
1272 +       int ret;
1273 +
1274 +       nand->controller = &nfc->controller;
1275 +       nand_set_controller_data(nand, (void *)nfc);
1276 +       nand_set_flash_node(nand, nfc->dev->of_node);
1277 +
1278 +       nand->options |= NAND_USES_DMA | NAND_NO_SUBPAGE_WRITE;
1279 +       if (!nfc->nfi_clk)
1280 +               nand->options |= NAND_KEEP_TIMINGS;
1281 +
1282 +       nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1283 +       nand->ecc.read_page = mt7621_nfc_read_page_hwecc;
1284 +       nand->ecc.read_page_raw = mt7621_nfc_read_page_raw;
1285 +       nand->ecc.write_page = mt7621_nfc_write_page_hwecc;
1286 +       nand->ecc.write_page_raw = mt7621_nfc_write_page_raw;
1287 +       nand->ecc.read_oob = mt7621_nfc_read_oob_hwecc;
1288 +       nand->ecc.read_oob_raw = mt7621_nfc_read_oob_raw;
1289 +       nand->ecc.write_oob = mt7621_nfc_write_oob_hwecc;
1290 +       nand->ecc.write_oob_raw = mt7621_nfc_write_oob_raw;
1291 +
1292 +       mtd = nand_to_mtd(nand);
1293 +       mtd->owner = THIS_MODULE;
1294 +       mtd->dev.parent = nfc->dev;
1295 +       mtd->name = MT7621_NFC_NAME;
1296 +       mtd_set_ooblayout(mtd, &mt7621_nfc_ooblayout_ops);
1297 +
1298 +       mt7621_nfc_hw_init(nfc);
1299 +
1300 +       ret = nand_scan(nand, 1);
1301 +       if (ret)
1302 +               return ret;
1303 +
1304 +       ret = mtd_device_register(mtd, NULL, 0);
1305 +       if (ret) {
1306 +               dev_err(nfc->dev, "Failed to register MTD: %d\n", ret);
1307 +               nand_cleanup(nand);
1308 +               return ret;
1309 +       }
1310 +
1311 +       return 0;
1312 +}
1313 +
1314 +static int mt7621_nfc_probe(struct platform_device *pdev)
1315 +{
1316 +       struct device *dev = &pdev->dev;
1317 +       struct mt7621_nfc *nfc;
1318 +       struct resource *res;
1319 +       int ret;
1320 +
1321 +       nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1322 +       if (!nfc)
1323 +               return -ENOMEM;
1324 +
1325 +       nand_controller_init(&nfc->controller);
1326 +       nfc->controller.ops = &mt7621_nfc_controller_ops;
1327 +       nfc->dev = dev;
1328 +
1329 +       res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nfi");
1330 +       nfc->nfi_regs = devm_ioremap_resource(dev, res);
1331 +       if (IS_ERR(nfc->nfi_regs)) {
1332 +               ret = PTR_ERR(nfc->nfi_regs);
1333 +               return ret;
1334 +       }
1335 +
1336 +       res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ecc");
1337 +       nfc->ecc_regs = devm_ioremap_resource(dev, res);
1338 +       if (IS_ERR(nfc->ecc_regs)) {
1339 +               ret = PTR_ERR(nfc->ecc_regs);
1340 +               return ret;
1341 +       }
1342 +
1343 +       nfc->nfi_clk = devm_clk_get(dev, "nfi_clk");
1344 +       if (IS_ERR(nfc->nfi_clk)) {
1345 +               dev_warn(dev, "nfi clk not provided\n");
1346 +               nfc->nfi_clk = NULL;
1347 +       } else {
1348 +               ret = clk_prepare_enable(nfc->nfi_clk);
1349 +               if (ret) {
1350 +                       dev_err(dev, "Failed to enable nfi core clock\n");
1351 +                       return ret;
1352 +               }
1353 +       }
1354 +
1355 +       platform_set_drvdata(pdev, nfc);
1356 +
1357 +       ret = mt7621_nfc_init_chip(nfc);
1358 +       if (ret) {
1359 +               dev_err(dev, "Failed to initialize nand chip\n");
1360 +               goto clk_disable;
1361 +       }
1362 +
1363 +       return 0;
1364 +
1365 +clk_disable:
1366 +       clk_disable_unprepare(nfc->nfi_clk);
1367 +
1368 +       return ret;
1369 +}
1370 +
1371 +static int mt7621_nfc_remove(struct platform_device *pdev)
1372 +{
1373 +       struct mt7621_nfc *nfc = platform_get_drvdata(pdev);
1374 +       struct nand_chip *nand = &nfc->nand;
1375 +       struct mtd_info *mtd = nand_to_mtd(nand);
1376 +
1377 +       mtd_device_unregister(mtd);
1378 +       nand_cleanup(nand);
1379 +       clk_disable_unprepare(nfc->nfi_clk);
1380 +
1381 +       return 0;
1382 +}
1383 +
1384 +static const struct of_device_id mt7621_nfc_id_table[] = {
1385 +       { .compatible = "mediatek,mt7621-nfc" },
1386 +       { },
1387 +};
1388 +MODULE_DEVICE_TABLE(of, match);
1389 +
1390 +static struct platform_driver mt7621_nfc_driver = {
1391 +       .probe = mt7621_nfc_probe,
1392 +       .remove = mt7621_nfc_remove,
1393 +       .driver = {
1394 +               .name = MT7621_NFC_NAME,
1395 +               .owner = THIS_MODULE,
1396 +               .of_match_table = mt7621_nfc_id_table,
1397 +       },
1398 +};
1399 +module_platform_driver(mt7621_nfc_driver);
1400 +
1401 +MODULE_LICENSE("GPL");
1402 +MODULE_AUTHOR("Weijie Gao <weijie.gao@mediatek.com>");
1403 +MODULE_DESCRIPTION("MediaTek MT7621 NAND Flash Controller driver");