target/linux/mediatek/files-5.10/drivers/mtd/mtk-snand/mtk-snand-ecc.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /*
   3  * Copyright (C) 2020 MediaTek Inc. All Rights Reserved.
   4  *
   5  * Author: Weijie Gao <weijie.gao@mediatek.com>
   6  */
   7
   8 #include "mtk-snand-def.h"
   9
  10 /* ECC registers */
  11 #define ECC_ENCCON                      0x000
  12 #define ENC_EN                          BIT(0)
  13
  14 #define ECC_ENCCNFG                     0x004
  15 #define ENC_MS_S                        16
  16 #define ENC_BURST_EN                    BIT(8)
  17 #define ENC_TNUM_S                      0
  18
  19 #define ECC_ENCIDLE                     0x00c
  20 #define ENC_IDLE                        BIT(0)
  21
  22 #define ECC_DECCON                      0x100
  23 #define DEC_EN                          BIT(0)
  24
  25 #define ECC_DECCNFG                     0x104
  26 #define DEC_EMPTY_EN                    BIT(31)
  27 #define DEC_CS_S                        16
  28 #define DEC_CON_S                       12
  29 #define   DEC_CON_CORRECT               3
  30 #define DEC_BURST_EN                    BIT(8)
  31 #define DEC_TNUM_S                      0
  32
  33 #define ECC_DECIDLE                     0x10c
  34 #define DEC_IDLE                        BIT(0)
  35
  36 #define ECC_DECENUM0                    0x114
  37 #define ECC_DECENUM(n)                  (ECC_DECENUM0 + (n) * 4)
  38
  39 /* ECC_ENCIDLE & ECC_DECIDLE */
  40 #define ECC_IDLE                        BIT(0)
  41
  42 /* ENC_MODE & DEC_MODE */
  43 #define ECC_MODE_NFI                    1
  44
  45 #define ECC_TIMEOUT                     500000
  46
  47 static const uint8_t mt7622_ecc_caps[] = { 4, 6, 8, 10, 12 };
  48
  49 static const uint32_t mt7622_ecc_regs[] = {
  50         [ECC_DECDONE] = 0x11c,
  51 };
  52
  53 static const struct mtk_ecc_soc_data mtk_ecc_socs[__SNAND_SOC_MAX] = {
  54         [SNAND_SOC_MT7622] = {
  55                 .ecc_caps = mt7622_ecc_caps,
  56                 .num_ecc_cap = ARRAY_SIZE(mt7622_ecc_caps),
  57                 .regs = mt7622_ecc_regs,
  58                 .mode_shift = 4,
  59                 .errnum_bits = 5,
  60                 .errnum_shift = 5,
  61         },
  62         [SNAND_SOC_MT7629] = {
  63                 .ecc_caps = mt7622_ecc_caps,
  64                 .num_ecc_cap = ARRAY_SIZE(mt7622_ecc_caps),
  65                 .regs = mt7622_ecc_regs,
  66                 .mode_shift = 4,
  67                 .errnum_bits = 5,
  68                 .errnum_shift = 5,
  69         },
  70 };
  71
  72 static inline uint32_t ecc_read32(struct mtk_snand *snf, uint32_t reg)
  73 {
  74         return readl(snf->ecc_base + reg);
  75 }
  76
  77 static inline void ecc_write32(struct mtk_snand *snf, uint32_t reg,
  78                                uint32_t val)
  79 {
  80         writel(val, snf->ecc_base + reg);
  81 }
  82
  83 static inline void ecc_write16(struct mtk_snand *snf, uint32_t reg,
  84                                uint16_t val)
  85 {
  86         writew(val, snf->ecc_base + reg);
  87 }
  88
  89 static int mtk_ecc_poll(struct mtk_snand *snf, uint32_t reg, uint32_t bits)
  90 {
  91         uint32_t val;
  92
  93         return read16_poll_timeout(snf->ecc_base + reg, val, (val & bits), 0,
  94                                    ECC_TIMEOUT);
  95 }
  96
  97 static int mtk_ecc_wait_idle(struct mtk_snand *snf, uint32_t reg)
  98 {
  99         int ret;
 100
 101         ret = mtk_ecc_poll(snf, reg, ECC_IDLE);
 102         if (ret) {
 103                 snand_log_ecc(snf->pdev, "ECC engine is busy\n");
 104                 return -EBUSY;
 105         }
 106
 107         return 0;
 108 }
 109
 110 int mtk_ecc_setup(struct mtk_snand *snf, void *fmdaddr, uint32_t max_ecc_bytes,
 111                   uint32_t msg_size)
 112 {
 113         uint32_t i, val, ecc_msg_bits, ecc_strength;
 114         int ret;
 115
 116         snf->ecc_soc = &mtk_ecc_socs[snf->soc];
 117
 118         snf->ecc_parity_bits = fls(1 + 8 * msg_size);
 119         ecc_strength = max_ecc_bytes * 8 / snf->ecc_parity_bits;
 120
 121         for (i = snf->ecc_soc->num_ecc_cap - 1; i >= 0; i--) {
 122                 if (snf->ecc_soc->ecc_caps[i] <= ecc_strength)
 123                         break;
 124         }
 125
 126         if (unlikely(i < 0)) {
 127                 snand_log_ecc(snf->pdev, "Page size %u+%u is not supported\n",
 128                               snf->writesize, snf->oobsize);
 129                 return -ENOTSUPP;
 130         }
 131
 132         snf->ecc_strength = snf->ecc_soc->ecc_caps[i];
 133         snf->ecc_bytes = DIV_ROUND_UP(snf->ecc_strength * snf->ecc_parity_bits,
 134                                       8);
 135
 136         /* Encoder config */
 137         ecc_write16(snf, ECC_ENCCON, 0);
 138         ret = mtk_ecc_wait_idle(snf, ECC_ENCIDLE);
 139         if (ret)
 140                 return ret;
 141
 142         ecc_msg_bits = msg_size * 8;
 143         val = (ecc_msg_bits << ENC_MS_S) |
 144               (ECC_MODE_NFI << snf->ecc_soc->mode_shift) | i;
 145         ecc_write32(snf, ECC_ENCCNFG, val);
 146
 147         /* Decoder config */
 148         ecc_write16(snf, ECC_DECCON, 0);
 149         ret = mtk_ecc_wait_idle(snf, ECC_DECIDLE);
 150         if (ret)
 151                 return ret;
 152
 153         ecc_msg_bits += snf->ecc_strength * snf->ecc_parity_bits;
 154         val = DEC_EMPTY_EN | (ecc_msg_bits << DEC_CS_S) |
 155               (DEC_CON_CORRECT << DEC_CON_S) |
 156               (ECC_MODE_NFI << snf->ecc_soc->mode_shift) | i;
 157         ecc_write32(snf, ECC_DECCNFG, val);
 158
 159         return 0;
 160 }
 161
 162 int mtk_snand_ecc_encoder_start(struct mtk_snand *snf)
 163 {
 164         int ret;
 165
 166         ret = mtk_ecc_wait_idle(snf, ECC_ENCIDLE);
 167         if (ret) {
 168                 ecc_write16(snf, ECC_ENCCON, 0);
 169                 mtk_ecc_wait_idle(snf, ECC_ENCIDLE);
 170         }
 171
 172         ecc_write16(snf, ECC_ENCCON, ENC_EN);
 173
 174         return 0;
 175 }
 176
 177 void mtk_snand_ecc_encoder_stop(struct mtk_snand *snf)
 178 {
 179         mtk_ecc_wait_idle(snf, ECC_ENCIDLE);
 180         ecc_write16(snf, ECC_ENCCON, 0);
 181 }
 182
 183 int mtk_snand_ecc_decoder_start(struct mtk_snand *snf)
 184 {
 185         int ret;
 186
 187         ret = mtk_ecc_wait_idle(snf, ECC_DECIDLE);
 188         if (ret) {
 189                 ecc_write16(snf, ECC_DECCON, 0);
 190                 mtk_ecc_wait_idle(snf, ECC_DECIDLE);
 191         }
 192
 193         ecc_write16(snf, ECC_DECCON, DEC_EN);
 194
 195         return 0;
 196 }
 197
 198 void mtk_snand_ecc_decoder_stop(struct mtk_snand *snf)
 199 {
 200         mtk_ecc_wait_idle(snf, ECC_DECIDLE);
 201         ecc_write16(snf, ECC_DECCON, 0);
 202 }
 203
 204 int mtk_ecc_wait_decoder_done(struct mtk_snand *snf)
 205 {
 206         uint16_t val, step_mask = (1 << snf->ecc_steps) - 1;
 207         uint32_t reg = snf->ecc_soc->regs[ECC_DECDONE];
 208         int ret;
 209
 210         ret = read16_poll_timeout(snf->ecc_base + reg, val,
 211                                   (val & step_mask) == step_mask, 0,
 212                                   ECC_TIMEOUT);
 213         if (ret)
 214                 snand_log_ecc(snf->pdev, "ECC decoder is busy\n");
 215
 216         return ret;
 217 }
 218
 219 int mtk_ecc_check_decode_error(struct mtk_snand *snf)
 220 {
 221         uint32_t i, regi, fi, errnum;
 222         uint32_t errnum_shift = snf->ecc_soc->errnum_shift;
 223         uint32_t errnum_mask = (1 << snf->ecc_soc->errnum_bits) - 1;
 224         int ret = 0;
 225
 226         for (i = 0; i < snf->ecc_steps; i++) {
 227                 regi = i / 4;
 228                 fi = i % 4;
 229
 230                 errnum = ecc_read32(snf, ECC_DECENUM(regi));
 231                 errnum = (errnum >> (fi * errnum_shift)) & errnum_mask;
 232
 233                 if (errnum <= snf->ecc_strength) {
 234                         snf->sect_bf[i] = errnum;
 235                 } else {
 236                         snf->sect_bf[i] = -1;
 237                         ret = -EBADMSG;
 238                 }
 239         }
 240
 241         return ret;
 242 }
 243
 244 static int mtk_ecc_check_buf_bitflips(struct mtk_snand *snf, const void *buf,
 245                                       size_t len, uint32_t bitflips)
 246 {
 247         const uint8_t *buf8 = buf;
 248         const uint32_t *buf32;
 249         uint32_t d, weight;
 250
 251         while (len && ((uintptr_t)buf8) % sizeof(uint32_t)) {
 252                 weight = hweight8(*buf8);
 253                 bitflips += BITS_PER_BYTE - weight;
 254                 buf8++;
 255                 len--;
 256
 257                 if (bitflips > snf->ecc_strength)
 258                         return -EBADMSG;
 259         }
 260
 261         buf32 = (const uint32_t *)buf8;
 262         while (len >= sizeof(uint32_t)) {
 263                 d = *buf32;
 264
 265                 if (d != ~0) {
 266                         weight = hweight32(d);
 267                         bitflips += sizeof(uint32_t) * BITS_PER_BYTE - weight;
 268                 }
 269
 270                 buf32++;
 271                 len -= sizeof(uint32_t);
 272
 273                 if (bitflips > snf->ecc_strength)
 274                         return -EBADMSG;
 275         }
 276
 277         buf8 = (const uint8_t *)buf32;
 278         while (len) {
 279                 weight = hweight8(*buf8);
 280                 bitflips += BITS_PER_BYTE - weight;
 281                 buf8++;
 282                 len--;
 283
 284                 if (bitflips > snf->ecc_strength)
 285                         return -EBADMSG;
 286         }
 287
 288         return bitflips;
 289 }
 290
 291 static int mtk_ecc_check_parity_bitflips(struct mtk_snand *snf, const void *buf,
 292                                          uint32_t bits, uint32_t bitflips)
 293 {
 294         uint32_t len, i;
 295         uint8_t b;
 296         int rc;
 297
 298         len = bits >> 3;
 299         bits &= 7;
 300
 301         rc = mtk_ecc_check_buf_bitflips(snf, buf, len, bitflips);
 302         if (!bits || rc < 0)
 303                 return rc;
 304
 305         bitflips = rc;
 306
 307         /* We want a precise count of bits */
 308         b = ((const uint8_t *)buf)[len];
 309         for (i = 0; i < bits; i++) {
 310                 if (!(b & BIT(i)))
 311                         bitflips++;
 312         }
 313
 314         if (bitflips > snf->ecc_strength)
 315                 return -EBADMSG;
 316
 317         return bitflips;
 318 }
 319
 320 static void mtk_ecc_reset_parity(void *buf, uint32_t bits)
 321 {
 322         uint32_t len;
 323
 324         len = bits >> 3;
 325         bits &= 7;
 326
 327         memset(buf, 0xff, len);
 328
 329         /* Only reset bits protected by ECC to 1 */
 330         if (bits)
 331                 ((uint8_t *)buf)[len] |= GENMASK(bits - 1, 0);
 332 }
 333
 334 int mtk_ecc_fixup_empty_sector(struct mtk_snand *snf, uint32_t sect)
 335 {
 336         uint32_t ecc_bytes = snf->spare_per_sector - snf->nfi_soc->fdm_size;
 337         uint8_t *oob = snf->page_cache + snf->writesize;
 338         uint8_t *data_ptr, *fdm_ptr, *ecc_ptr;
 339         int bitflips = 0, ecc_bits, parity_bits;
 340
 341         parity_bits = fls(snf->nfi_soc->sector_size * 8);
 342         ecc_bits = snf->ecc_strength * parity_bits;
 343
 344         data_ptr = snf->page_cache + sect * snf->nfi_soc->sector_size;
 345         fdm_ptr = oob + sect * snf->nfi_soc->fdm_size;
 346         ecc_ptr = oob + snf->ecc_steps * snf->nfi_soc->fdm_size +
 347                   sect * ecc_bytes;
 348
 349         /*
 350          * Check whether DATA + FDM + ECC of a sector contains correctable
 351          * bitflips
 352          */
 353         bitflips = mtk_ecc_check_buf_bitflips(snf, data_ptr,
 354                                               snf->nfi_soc->sector_size,
 355                                               bitflips);
 356         if (bitflips < 0)
 357                 return -EBADMSG;
 358
 359         bitflips = mtk_ecc_check_buf_bitflips(snf, fdm_ptr,
 360                                               snf->nfi_soc->fdm_ecc_size,
 361                                               bitflips);
 362         if (bitflips < 0)
 363                 return -EBADMSG;
 364
 365         bitflips = mtk_ecc_check_parity_bitflips(snf, ecc_ptr, ecc_bits,
 366                                                  bitflips);
 367         if (bitflips < 0)
 368                 return -EBADMSG;
 369
 370         if (!bitflips)
 371                 return 0;
 372
 373         /* Reset the data of this sector to 0xff */
 374         memset(data_ptr, 0xff, snf->nfi_soc->sector_size);
 375         memset(fdm_ptr, 0xff, snf->nfi_soc->fdm_ecc_size);
 376         mtk_ecc_reset_parity(ecc_ptr, ecc_bits);
 377
 378         return bitflips;
 379 }