drivers/mtd/nand/raw/omap_gpmc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * (C) Copyright 2004-2008 Texas Instruments, <www.ti.com>
   4  * Rohit Choraria <rohitkc@ti.com>
   5  */
   6
   7 #include <common.h>
   8 #include <log.h>
   9 #include <asm/io.h>
  10 #include <linux/errno.h>
  11 #include <asm/arch/mem.h>
  12 #include <linux/mtd/omap_gpmc.h>
  13 #include <linux/mtd/nand_ecc.h>
  14 #include <linux/mtd/rawnand.h>
  15 #include <linux/bch.h>
  16 #include <linux/compiler.h>
  17 #include <nand.h>
  18 #include <linux/mtd/omap_elm.h>
  19
  20 #define BADBLOCK_MARKER_LENGTH  2
  21 #define SECTOR_BYTES            512
  22 #define ECCCLEAR                (0x1 << 8)
  23 #define ECCRESULTREG1           (0x1 << 0)
  24 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
  25 #define BCH4_BIT_PAD            4
  26
  27 #ifdef CONFIG_BCH
  28 static u8  bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
  29                                 0x97, 0x79, 0xe5, 0x24, 0xb5};
  30 #endif
  31 static uint8_t cs_next;
  32 static __maybe_unused struct nand_ecclayout omap_ecclayout;
  33
  34 #if defined(CONFIG_NAND_OMAP_GPMC_WSCFG)
  35 static const int8_t wscfg[CONFIG_SYS_MAX_NAND_DEVICE] =
  36         { CONFIG_NAND_OMAP_GPMC_WSCFG };
  37 #else
  38 /* wscfg is preset to zero since its a static variable */
  39 static const int8_t wscfg[CONFIG_SYS_MAX_NAND_DEVICE];
  40 #endif
  41
  42 /*
  43  * Driver configurations
  44  */
  45 struct omap_nand_info {
  46         struct bch_control *control;
  47         enum omap_ecc ecc_scheme;
  48         uint8_t cs;
  49         uint8_t ws;             /* wait status pin (0,1) */
  50 };
  51
  52 /* We are wasting a bit of memory but al least we are safe */
  53 static struct omap_nand_info omap_nand_info[GPMC_MAX_CS];
  54
  55 /*
  56  * omap_nand_hwcontrol - Set the address pointers corretly for the
  57  *                      following address/data/command operation
  58  */
  59 static void omap_nand_hwcontrol(struct mtd_info *mtd, int32_t cmd,
  60                                 uint32_t ctrl)
  61 {
  62         register struct nand_chip *this = mtd_to_nand(mtd);
  63         struct omap_nand_info *info = nand_get_controller_data(this);
  64         int cs = info->cs;
  65
  66         /*
  67          * Point the IO_ADDR to DATA and ADDRESS registers instead
  68          * of chip address
  69          */
  70         switch (ctrl) {
  71         case NAND_CTRL_CHANGE | NAND_CTRL_CLE:
  72                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
  73                 break;
  74         case NAND_CTRL_CHANGE | NAND_CTRL_ALE:
  75                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_adr;
  76                 break;
  77         case NAND_CTRL_CHANGE | NAND_NCE:
  78                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
  79                 break;
  80         }
  81
  82         if (cmd != NAND_CMD_NONE)
  83                 writeb(cmd, this->IO_ADDR_W);
  84 }
  85
  86 /* Check wait pin as dev ready indicator */
  87 static int omap_dev_ready(struct mtd_info *mtd)
  88 {
  89         register struct nand_chip *this = mtd_to_nand(mtd);
  90         struct omap_nand_info *info = nand_get_controller_data(this);
  91         return gpmc_cfg->status & (1 << (8 + info->ws));
  92 }
  93
  94 /*
  95  * gen_true_ecc - This function will generate true ECC value, which
  96  * can be used when correcting data read from NAND flash memory core
  97  *
  98  * @ecc_buf:    buffer to store ecc code
  99  *
 100  * @return:     re-formatted ECC value
 101  */
 102 static uint32_t gen_true_ecc(uint8_t *ecc_buf)
 103 {
 104         return ecc_buf[0] | (ecc_buf[1] << 16) | ((ecc_buf[2] & 0xF0) << 20) |
 105                 ((ecc_buf[2] & 0x0F) << 8);
 106 }
 107
 108 /*
 109  * omap_correct_data - Compares the ecc read from nand spare area with ECC
 110  * registers values and corrects one bit error if it has occurred
 111  * Further details can be had from OMAP TRM and the following selected links:
 112  * http://en.wikipedia.org/wiki/Hamming_code
 113  * http://www.cs.utexas.edu/users/plaxton/c/337/05f/slides/ErrorCorrection-4.pdf
 114  *
 115  * @mtd:                 MTD device structure
 116  * @dat:                 page data
 117  * @read_ecc:            ecc read from nand flash
 118  * @calc_ecc:            ecc read from ECC registers
 119  *
 120  * Return: 0 if data is OK or corrected, else returns -1
 121  */
 122 static int __maybe_unused omap_correct_data(struct mtd_info *mtd, uint8_t *dat,
 123                                 uint8_t *read_ecc, uint8_t *calc_ecc)
 124 {
 125         uint32_t orig_ecc, new_ecc, res, hm;
 126         uint16_t parity_bits, byte;
 127         uint8_t bit;
 128
 129         /* Regenerate the orginal ECC */
 130         orig_ecc = gen_true_ecc(read_ecc);
 131         new_ecc = gen_true_ecc(calc_ecc);
 132         /* Get the XOR of real ecc */
 133         res = orig_ecc ^ new_ecc;
 134         if (res) {
 135                 /* Get the hamming width */
 136                 hm = hweight32(res);
 137                 /* Single bit errors can be corrected! */
 138                 if (hm == 12) {
 139                         /* Correctable data! */
 140                         parity_bits = res >> 16;
 141                         bit = (parity_bits & 0x7);
 142                         byte = (parity_bits >> 3) & 0x1FF;
 143                         /* Flip the bit to correct */
 144                         dat[byte] ^= (0x1 << bit);
 145                 } else if (hm == 1) {
 146                         printf("Error: Ecc is wrong\n");
 147                         /* ECC itself is corrupted */
 148                         return 2;
 149                 } else {
 150                         /*
 151                          * hm distance != parity pairs OR one, could mean 2 bit
 152                          * error OR potentially be on a blank page..
 153                          * orig_ecc: contains spare area data from nand flash.
 154                          * new_ecc: generated ecc while reading data area.
 155                          * Note: if the ecc = 0, all data bits from which it was
 156                          * generated are 0xFF.
 157                          * The 3 byte(24 bits) ecc is generated per 512byte
 158                          * chunk of a page. If orig_ecc(from spare area)
 159                          * is 0xFF && new_ecc(computed now from data area)=0x0,
 160                          * this means that data area is 0xFF and spare area is
 161                          * 0xFF. A sure sign of a erased page!
 162                          */
 163                         if ((orig_ecc == 0x0FFF0FFF) && (new_ecc == 0x00000000))
 164                                 return 0;
 165                         printf("Error: Bad compare! failed\n");
 166                         /* detected 2 bit error */
 167                         return -EBADMSG;
 168                 }
 169         }
 170         return 0;
 171 }
 172
 173 /*
 174  * omap_enable_hwecc - configures GPMC as per ECC scheme before read/write
 175  * @mtd:        MTD device structure
 176  * @mode:       Read/Write mode
 177  */
 178 __maybe_unused
 179 static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode)
 180 {
 181         struct nand_chip        *nand   = mtd_to_nand(mtd);
 182         struct omap_nand_info   *info   = nand_get_controller_data(nand);
 183         unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0;
 184         unsigned int ecc_algo = 0;
 185         unsigned int bch_type = 0;
 186         unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00;
 187         u32 ecc_size_config_val = 0;
 188         u32 ecc_config_val = 0;
 189         int cs = info->cs;
 190
 191         /* configure GPMC for specific ecc-scheme */
 192         switch (info->ecc_scheme) {
 193         case OMAP_ECC_HAM1_CODE_SW:
 194                 return;
 195         case OMAP_ECC_HAM1_CODE_HW:
 196                 ecc_algo = 0x0;
 197                 bch_type = 0x0;
 198                 bch_wrapmode = 0x00;
 199                 eccsize0 = 0xFF;
 200                 eccsize1 = 0xFF;
 201                 break;
 202         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 203         case OMAP_ECC_BCH8_CODE_HW:
 204                 ecc_algo = 0x1;
 205                 bch_type = 0x1;
 206                 if (mode == NAND_ECC_WRITE) {
 207                         bch_wrapmode = 0x01;
 208                         eccsize0 = 0;  /* extra bits in nibbles per sector */
 209                         eccsize1 = 28; /* OOB bits in nibbles per sector */
 210                 } else {
 211                         bch_wrapmode = 0x01;
 212                         eccsize0 = 26; /* ECC bits in nibbles per sector */
 213                         eccsize1 = 2;  /* non-ECC bits in nibbles per sector */
 214                 }
 215                 break;
 216         case OMAP_ECC_BCH16_CODE_HW:
 217                 ecc_algo = 0x1;
 218                 bch_type = 0x2;
 219                 if (mode == NAND_ECC_WRITE) {
 220                         bch_wrapmode = 0x01;
 221                         eccsize0 = 0;  /* extra bits in nibbles per sector */
 222                         eccsize1 = 52; /* OOB bits in nibbles per sector */
 223                 } else {
 224                         bch_wrapmode = 0x01;
 225                         eccsize0 = 52; /* ECC bits in nibbles per sector */
 226                         eccsize1 = 0;  /* non-ECC bits in nibbles per sector */
 227                 }
 228                 break;
 229         default:
 230                 return;
 231         }
 232         /* Clear ecc and enable bits */
 233         writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
 234         /* Configure ecc size for BCH */
 235         ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12);
 236         writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config);
 237
 238         /* Configure device details for BCH engine */
 239         ecc_config_val = ((ecc_algo << 16)      | /* HAM1 | BCHx */
 240                         (bch_type << 12)        | /* BCH4/BCH8/BCH16 */
 241                         (bch_wrapmode << 8)     | /* wrap mode */
 242                         (dev_width << 7)        | /* bus width */
 243                         (0x0 << 4)              | /* number of sectors */
 244                         (cs <<  1)              | /* ECC CS */
 245                         (0x1));                   /* enable ECC */
 246         writel(ecc_config_val, &gpmc_cfg->ecc_config);
 247 }
 248
 249 /*
 250  *  omap_calculate_ecc - Read ECC result
 251  *  @mtd:       MTD structure
 252  *  @dat:       unused
 253  *  @ecc_code:  ecc_code buffer
 254  *  Using noninverted ECC can be considered ugly since writing a blank
 255  *  page ie. padding will clear the ECC bytes. This is no problem as
 256  *  long nobody is trying to write data on the seemingly unused page.
 257  *  Reading an erased page will produce an ECC mismatch between
 258  *  generated and read ECC bytes that has to be dealt with separately.
 259  *  E.g. if page is 0xFF (fresh erased), and if HW ECC engine within GPMC
 260  *  is used, the result of read will be 0x0 while the ECC offsets of the
 261  *  spare area will be 0xFF which will result in an ECC mismatch.
 262  */
 263 static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat,
 264                                 uint8_t *ecc_code)
 265 {
 266         struct nand_chip *chip = mtd_to_nand(mtd);
 267         struct omap_nand_info *info = nand_get_controller_data(chip);
 268         const uint32_t *ptr;
 269         uint32_t val = 0;
 270         int8_t i = 0, j;
 271
 272         switch (info->ecc_scheme) {
 273         case OMAP_ECC_HAM1_CODE_HW:
 274                 val = readl(&gpmc_cfg->ecc1_result);
 275                 ecc_code[0] = val & 0xFF;
 276                 ecc_code[1] = (val >> 16) & 0xFF;
 277                 ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
 278                 break;
 279 #ifdef CONFIG_BCH
 280         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 281 #endif
 282         case OMAP_ECC_BCH8_CODE_HW:
 283                 ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3];
 284                 val = readl(ptr);
 285                 ecc_code[i++] = (val >>  0) & 0xFF;
 286                 ptr--;
 287                 for (j = 0; j < 3; j++) {
 288                         val = readl(ptr);
 289                         ecc_code[i++] = (val >> 24) & 0xFF;
 290                         ecc_code[i++] = (val >> 16) & 0xFF;
 291                         ecc_code[i++] = (val >>  8) & 0xFF;
 292                         ecc_code[i++] = (val >>  0) & 0xFF;
 293                         ptr--;
 294                 }
 295                 break;
 296         case OMAP_ECC_BCH16_CODE_HW:
 297                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]);
 298                 ecc_code[i++] = (val >>  8) & 0xFF;
 299                 ecc_code[i++] = (val >>  0) & 0xFF;
 300                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]);
 301                 ecc_code[i++] = (val >> 24) & 0xFF;
 302                 ecc_code[i++] = (val >> 16) & 0xFF;
 303                 ecc_code[i++] = (val >>  8) & 0xFF;
 304                 ecc_code[i++] = (val >>  0) & 0xFF;
 305                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]);
 306                 ecc_code[i++] = (val >> 24) & 0xFF;
 307                 ecc_code[i++] = (val >> 16) & 0xFF;
 308                 ecc_code[i++] = (val >>  8) & 0xFF;
 309                 ecc_code[i++] = (val >>  0) & 0xFF;
 310                 for (j = 3; j >= 0; j--) {
 311                         val = readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j]
 312                                                                         );
 313                         ecc_code[i++] = (val >> 24) & 0xFF;
 314                         ecc_code[i++] = (val >> 16) & 0xFF;
 315                         ecc_code[i++] = (val >>  8) & 0xFF;
 316                         ecc_code[i++] = (val >>  0) & 0xFF;
 317                 }
 318                 break;
 319         default:
 320                 return -EINVAL;
 321         }
 322         /* ECC scheme specific syndrome customizations */
 323         switch (info->ecc_scheme) {
 324         case OMAP_ECC_HAM1_CODE_HW:
 325                 break;
 326 #ifdef CONFIG_BCH
 327         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 328
 329                 for (i = 0; i < chip->ecc.bytes; i++)
 330                         *(ecc_code + i) = *(ecc_code + i) ^
 331                                                 bch8_polynomial[i];
 332                 break;
 333 #endif
 334         case OMAP_ECC_BCH8_CODE_HW:
 335                 ecc_code[chip->ecc.bytes - 1] = 0x00;
 336                 break;
 337         case OMAP_ECC_BCH16_CODE_HW:
 338                 break;
 339         default:
 340                 return -EINVAL;
 341         }
 342         return 0;
 343 }
 344
 345 #ifdef CONFIG_NAND_OMAP_GPMC_PREFETCH
 346
 347 #define PREFETCH_CONFIG1_CS_SHIFT       24
 348 #define PREFETCH_FIFOTHRESHOLD_MAX      0x40
 349 #define PREFETCH_FIFOTHRESHOLD(val)     ((val) << 8)
 350 #define PREFETCH_STATUS_COUNT(val)      (val & 0x00003fff)
 351 #define PREFETCH_STATUS_FIFO_CNT(val)   ((val >> 24) & 0x7F)
 352 #define ENABLE_PREFETCH                 (1 << 7)
 353
 354 /**
 355  * omap_prefetch_enable - configures and starts prefetch transfer
 356  * @fifo_th: fifo threshold to be used for read/ write
 357  * @count: number of bytes to be transferred
 358  * @is_write: prefetch read(0) or write post(1) mode
 359  * @cs: chip select to use
 360  */
 361 static int omap_prefetch_enable(int fifo_th, unsigned int count, int is_write, int cs)
 362 {
 363         uint32_t val;
 364
 365         if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX)
 366                 return -EINVAL;
 367
 368         if (readl(&gpmc_cfg->prefetch_control))
 369                 return -EBUSY;
 370
 371         /* Set the amount of bytes to be prefetched */
 372         writel(count, &gpmc_cfg->prefetch_config2);
 373
 374         val = (cs << PREFETCH_CONFIG1_CS_SHIFT) | (is_write & 1) |
 375                 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH;
 376         writel(val, &gpmc_cfg->prefetch_config1);
 377
 378         /*  Start the prefetch engine */
 379         writel(1, &gpmc_cfg->prefetch_control);
 380
 381         return 0;
 382 }
 383
 384 /**
 385  * omap_prefetch_reset - disables and stops the prefetch engine
 386  */
 387 static void omap_prefetch_reset(void)
 388 {
 389         writel(0, &gpmc_cfg->prefetch_control);
 390         writel(0, &gpmc_cfg->prefetch_config1);
 391 }
 392
 393 static int __read_prefetch_aligned(struct nand_chip *chip, uint32_t *buf, int len)
 394 {
 395         int ret;
 396         uint32_t cnt;
 397         struct omap_nand_info *info = nand_get_controller_data(chip);
 398
 399         ret = omap_prefetch_enable(PREFETCH_FIFOTHRESHOLD_MAX, len, 0, info->cs);
 400         if (ret < 0)
 401                 return ret;
 402
 403         do {
 404                 int i;
 405
 406                 cnt = readl(&gpmc_cfg->prefetch_status);
 407                 cnt = PREFETCH_STATUS_FIFO_CNT(cnt);
 408
 409                 for (i = 0; i < cnt / 4; i++) {
 410                         *buf++ = readl(CONFIG_SYS_NAND_BASE);
 411                         len -= 4;
 412                 }
 413         } while (len);
 414
 415         omap_prefetch_reset();
 416
 417         return 0;
 418 }
 419
 420 static inline void omap_nand_read(struct mtd_info *mtd, uint8_t *buf, int len)
 421 {
 422         struct nand_chip *chip = mtd_to_nand(mtd);
 423
 424         if (chip->options & NAND_BUSWIDTH_16)
 425                 nand_read_buf16(mtd, buf, len);
 426         else
 427                 nand_read_buf(mtd, buf, len);
 428 }
 429
 430 static void omap_nand_read_prefetch(struct mtd_info *mtd, uint8_t *buf, int len)
 431 {
 432         int ret;
 433         uint32_t head, tail;
 434         struct nand_chip *chip = mtd_to_nand(mtd);
 435
 436         /*
 437          * If the destination buffer is unaligned, start with reading
 438          * the overlap byte-wise.
 439          */
 440         head = ((uint32_t) buf) % 4;
 441         if (head) {
 442                 omap_nand_read(mtd, buf, head);
 443                 buf += head;
 444                 len -= head;
 445         }
 446
 447         /*
 448          * Only transfer multiples of 4 bytes in a pre-fetched fashion.
 449          * If there's a residue, care for it byte-wise afterwards.
 450          */
 451         tail = len % 4;
 452
 453         ret = __read_prefetch_aligned(chip, (uint32_t *)buf, len - tail);
 454         if (ret < 0) {
 455                 /* fallback in case the prefetch engine is busy */
 456                 omap_nand_read(mtd, buf, len);
 457         } else if (tail) {
 458                 buf += len - tail;
 459                 omap_nand_read(mtd, buf, tail);
 460         }
 461 }
 462 #endif /* CONFIG_NAND_OMAP_GPMC_PREFETCH */
 463
 464 #ifdef CONFIG_NAND_OMAP_ELM
 465 /*
 466  * omap_reverse_list - re-orders list elements in reverse order [internal]
 467  * @list:       pointer to start of list
 468  * @length:     length of list
 469 */
 470 static void omap_reverse_list(u8 *list, unsigned int length)
 471 {
 472         unsigned int i, j;
 473         unsigned int half_length = length / 2;
 474         u8 tmp;
 475         for (i = 0, j = length - 1; i < half_length; i++, j--) {
 476                 tmp = list[i];
 477                 list[i] = list[j];
 478                 list[j] = tmp;
 479         }
 480 }
 481
 482 /*
 483  * omap_correct_data_bch - Compares the ecc read from nand spare area
 484  * with ECC registers values and corrects one bit error if it has occurred
 485  *
 486  * @mtd:        MTD device structure
 487  * @dat:        page data
 488  * @read_ecc:   ecc read from nand flash (ignored)
 489  * @calc_ecc:   ecc read from ECC registers
 490  *
 491  * Return: 0 if data is OK or corrected, else returns -1
 492  */
 493 static int omap_correct_data_bch(struct mtd_info *mtd, uint8_t *dat,
 494                                 uint8_t *read_ecc, uint8_t *calc_ecc)
 495 {
 496         struct nand_chip *chip = mtd_to_nand(mtd);
 497         struct omap_nand_info *info = nand_get_controller_data(chip);
 498         struct nand_ecc_ctrl *ecc = &chip->ecc;
 499         uint32_t error_count = 0, error_max;
 500         uint32_t error_loc[ELM_MAX_ERROR_COUNT];
 501         enum bch_level bch_type;
 502         uint32_t i, ecc_flag = 0;
 503         uint8_t count;
 504         uint32_t byte_pos, bit_pos;
 505         int err = 0;
 506
 507         /* check calculated ecc */
 508         for (i = 0; i < ecc->bytes && !ecc_flag; i++) {
 509                 if (calc_ecc[i] != 0x00)
 510                         goto not_ecc_match;
 511         }
 512         return 0;
 513 not_ecc_match:
 514
 515         /* check for whether it's an erased-page */
 516         for (i = 0; i < ecc->bytes; i++) {
 517                 if (read_ecc[i] != 0xff)
 518                         goto not_erased;
 519         }
 520         for (i = 0; i < SECTOR_BYTES; i++) {
 521                 if (dat[i] != 0xff)
 522                         goto not_erased;
 523         }
 524         return 0;
 525 not_erased:
 526
 527         /*
 528          * Check for whether it's an erased page with a correctable
 529          * number of bitflips. Erased pages have all 1's in the data,
 530          * so we just compute the number of 0 bits in the data and
 531          * see if it's under the correction threshold.
 532          *
 533          * NOTE: The check for a perfect erased page above is faster for
 534          * the more common case, even though it's logically redundant.
 535          */
 536         for (i = 0; i < ecc->bytes; i++)
 537                 error_count += hweight8(~read_ecc[i]);
 538
 539         for (i = 0; i < SECTOR_BYTES; i++)
 540                 error_count += hweight8(~dat[i]);
 541
 542         if (error_count <= ecc->strength) {
 543                 memset(read_ecc, 0xFF, ecc->bytes);
 544                 memset(dat, 0xFF, SECTOR_BYTES);
 545                 debug("nand: %u bit-flip(s) corrected in erased page\n",
 546                       error_count);
 547                 return error_count;
 548         }
 549
 550         /*
 551          * while reading ECC result we read it in big endian.
 552          * Hence while loading to ELM we have rotate to get the right endian.
 553          */
 554         switch (info->ecc_scheme) {
 555         case OMAP_ECC_BCH8_CODE_HW:
 556                 bch_type = BCH_8_BIT;
 557                 omap_reverse_list(calc_ecc, ecc->bytes - 1);
 558                 break;
 559         case OMAP_ECC_BCH16_CODE_HW:
 560                 bch_type = BCH_16_BIT;
 561                 omap_reverse_list(calc_ecc, ecc->bytes);
 562                 break;
 563         default:
 564                 return -EINVAL;
 565         }
 566         /* use elm module to check for errors */
 567         elm_config(bch_type);
 568         error_count = 0;
 569         err = elm_check_error(calc_ecc, bch_type, &error_count, error_loc);
 570         if (err)
 571                 return err;
 572
 573         /* correct bch error */
 574         for (count = 0; count < error_count; count++) {
 575                 switch (info->ecc_scheme) {
 576                 case OMAP_ECC_BCH8_CODE_HW:
 577                         /* 14th byte in ECC is reserved to match ROM layout */
 578                         error_max = SECTOR_BYTES + (ecc->bytes - 1);
 579                         break;
 580                 case OMAP_ECC_BCH16_CODE_HW:
 581                         error_max = SECTOR_BYTES + ecc->bytes;
 582                         break;
 583                 default:
 584                         return -EINVAL;
 585                 }
 586                 byte_pos = error_max - (error_loc[count] / 8) - 1;
 587                 bit_pos  = error_loc[count] % 8;
 588                 if (byte_pos < SECTOR_BYTES) {
 589                         dat[byte_pos] ^= 1 << bit_pos;
 590                         debug("nand: bit-flip corrected @data=%d\n", byte_pos);
 591                 } else if (byte_pos < error_max) {
 592                         read_ecc[byte_pos - SECTOR_BYTES] ^= 1 << bit_pos;
 593                         debug("nand: bit-flip corrected @oob=%d\n", byte_pos -
 594                                                                 SECTOR_BYTES);
 595                 } else {
 596                         err = -EBADMSG;
 597                         printf("nand: error: invalid bit-flip location\n");
 598                 }
 599         }
 600         return (err) ? err : error_count;
 601 }
 602
 603 /**
 604  * omap_read_page_bch - hardware ecc based page read function
 605  * @mtd:        mtd info structure
 606  * @chip:       nand chip info structure
 607  * @buf:        buffer to store read data
 608  * @oob_required: caller expects OOB data read to chip->oob_poi
 609  * @page:       page number to read
 610  *
 611  */
 612 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 613                                 uint8_t *buf, int oob_required, int page)
 614 {
 615         int i, eccsize = chip->ecc.size;
 616         int eccbytes = chip->ecc.bytes;
 617         int eccsteps = chip->ecc.steps;
 618         uint8_t *p = buf;
 619         uint8_t *ecc_calc = chip->buffers->ecccalc;
 620         uint8_t *ecc_code = chip->buffers->ecccode;
 621         uint32_t *eccpos = chip->ecc.layout->eccpos;
 622         uint8_t *oob = chip->oob_poi;
 623         uint32_t data_pos;
 624         uint32_t oob_pos;
 625
 626         data_pos = 0;
 627         /* oob area start */
 628         oob_pos = (eccsize * eccsteps) + chip->ecc.layout->eccpos[0];
 629         oob += chip->ecc.layout->eccpos[0];
 630
 631         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize,
 632                                 oob += eccbytes) {
 633                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
 634                 /* read data */
 635                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1);
 636                 chip->read_buf(mtd, p, eccsize);
 637
 638                 /* read respective ecc from oob area */
 639                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
 640                 chip->read_buf(mtd, oob, eccbytes);
 641                 /* read syndrome */
 642                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
 643
 644                 data_pos += eccsize;
 645                 oob_pos += eccbytes;
 646         }
 647
 648         for (i = 0; i < chip->ecc.total; i++)
 649                 ecc_code[i] = chip->oob_poi[eccpos[i]];
 650
 651         eccsteps = chip->ecc.steps;
 652         p = buf;
 653
 654         for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 655                 int stat;
 656
 657                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
 658                 if (stat < 0)
 659                         mtd->ecc_stats.failed++;
 660                 else
 661                         mtd->ecc_stats.corrected += stat;
 662         }
 663         return 0;
 664 }
 665 #endif /* CONFIG_NAND_OMAP_ELM */
 666
 667 /*
 668  * OMAP3 BCH8 support (with BCH library)
 669  */
 670 #ifdef CONFIG_BCH
 671 /**
 672  * omap_correct_data_bch_sw - Decode received data and correct errors
 673  * @mtd: MTD device structure
 674  * @data: page data
 675  * @read_ecc: ecc read from nand flash
 676  * @calc_ecc: ecc read from HW ECC registers
 677  */
 678 static int omap_correct_data_bch_sw(struct mtd_info *mtd, u_char *data,
 679                                  u_char *read_ecc, u_char *calc_ecc)
 680 {
 681         int i, count;
 682         /* cannot correct more than 8 errors */
 683         unsigned int errloc[8];
 684         struct nand_chip *chip = mtd_to_nand(mtd);
 685         struct omap_nand_info *info = nand_get_controller_data(chip);
 686
 687         count = decode_bch(info->control, NULL, SECTOR_BYTES,
 688                                 read_ecc, calc_ecc, NULL, errloc);
 689         if (count > 0) {
 690                 /* correct errors */
 691                 for (i = 0; i < count; i++) {
 692                         /* correct data only, not ecc bytes */
 693                         if (errloc[i] < SECTOR_BYTES << 3)
 694                                 data[errloc[i] >> 3] ^= 1 << (errloc[i] & 7);
 695                         debug("corrected bitflip %u\n", errloc[i]);
 696 #ifdef DEBUG
 697                         puts("read_ecc: ");
 698                         /*
 699                          * BCH8 have 13 bytes of ECC; BCH4 needs adoption
 700                          * here!
 701                          */
 702                         for (i = 0; i < 13; i++)
 703                                 printf("%02x ", read_ecc[i]);
 704                         puts("\n");
 705                         puts("calc_ecc: ");
 706                         for (i = 0; i < 13; i++)
 707                                 printf("%02x ", calc_ecc[i]);
 708                         puts("\n");
 709 #endif
 710                 }
 711         } else if (count < 0) {
 712                 puts("ecc unrecoverable error\n");
 713         }
 714         return count;
 715 }
 716
 717 /**
 718  * omap_free_bch - Release BCH ecc resources
 719  * @mtd: MTD device structure
 720  */
 721 static void __maybe_unused omap_free_bch(struct mtd_info *mtd)
 722 {
 723         struct nand_chip *chip = mtd_to_nand(mtd);
 724         struct omap_nand_info *info = nand_get_controller_data(chip);
 725
 726         if (info->control) {
 727                 free_bch(info->control);
 728                 info->control = NULL;
 729         }
 730 }
 731 #endif /* CONFIG_BCH */
 732
 733 /**
 734  * omap_select_ecc_scheme - configures driver for particular ecc-scheme
 735  * @nand: NAND chip device structure
 736  * @ecc_scheme: ecc scheme to configure
 737  * @pagesize: number of main-area bytes per page of NAND device
 738  * @oobsize: number of OOB/spare bytes per page of NAND device
 739  */
 740 static int omap_select_ecc_scheme(struct nand_chip *nand,
 741         enum omap_ecc ecc_scheme, unsigned int pagesize, unsigned int oobsize) {
 742         struct omap_nand_info   *info           = nand_get_controller_data(nand);
 743         struct nand_ecclayout   *ecclayout      = &omap_ecclayout;
 744         int eccsteps = pagesize / SECTOR_BYTES;
 745         int i;
 746
 747         switch (ecc_scheme) {
 748         case OMAP_ECC_HAM1_CODE_SW:
 749                 debug("nand: selected OMAP_ECC_HAM1_CODE_SW\n");
 750                 /* For this ecc-scheme, ecc.bytes, ecc.layout, ... are
 751                  * initialized in nand_scan_tail(), so just set ecc.mode */
 752                 info->control           = NULL;
 753                 nand->ecc.mode          = NAND_ECC_SOFT;
 754                 nand->ecc.layout        = NULL;
 755                 nand->ecc.size          = 0;
 756                 break;
 757
 758         case OMAP_ECC_HAM1_CODE_HW:
 759                 debug("nand: selected OMAP_ECC_HAM1_CODE_HW\n");
 760                 /* check ecc-scheme requirements before updating ecc info */
 761                 if ((3 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 762                         printf("nand: error: insufficient OOB: require=%d\n", (
 763                                 (3 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 764                         return -EINVAL;
 765                 }
 766                 info->control           = NULL;
 767                 /* populate ecc specific fields */
 768                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 769                 nand->ecc.mode          = NAND_ECC_HW;
 770                 nand->ecc.strength      = 1;
 771                 nand->ecc.size          = SECTOR_BYTES;
 772                 nand->ecc.bytes         = 3;
 773                 nand->ecc.hwctl         = omap_enable_hwecc;
 774                 nand->ecc.correct       = omap_correct_data;
 775                 nand->ecc.calculate     = omap_calculate_ecc;
 776                 /* define ecc-layout */
 777                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 778                 for (i = 0; i < ecclayout->eccbytes; i++) {
 779                         if (nand->options & NAND_BUSWIDTH_16)
 780                                 ecclayout->eccpos[i] = i + 2;
 781                         else
 782                                 ecclayout->eccpos[i] = i + 1;
 783                 }
 784                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 785                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 786                                                 BADBLOCK_MARKER_LENGTH;
 787                 break;
 788
 789         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 790 #ifdef CONFIG_BCH
 791                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
 792                 /* check ecc-scheme requirements before updating ecc info */
 793                 if ((13 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 794                         printf("nand: error: insufficient OOB: require=%d\n", (
 795                                 (13 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 796                         return -EINVAL;
 797                 }
 798                 /* check if BCH S/W library can be used for error detection */
 799                 info->control = init_bch(13, 8, 0x201b);
 800                 if (!info->control) {
 801                         printf("nand: error: could not init_bch()\n");
 802                         return -ENODEV;
 803                 }
 804                 /* populate ecc specific fields */
 805                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 806                 nand->ecc.mode          = NAND_ECC_HW;
 807                 nand->ecc.strength      = 8;
 808                 nand->ecc.size          = SECTOR_BYTES;
 809                 nand->ecc.bytes         = 13;
 810                 nand->ecc.hwctl         = omap_enable_hwecc;
 811                 nand->ecc.correct       = omap_correct_data_bch_sw;
 812                 nand->ecc.calculate     = omap_calculate_ecc;
 813                 /* define ecc-layout */
 814                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 815                 ecclayout->eccpos[0]    = BADBLOCK_MARKER_LENGTH;
 816                 for (i = 1; i < ecclayout->eccbytes; i++) {
 817                         if (i % nand->ecc.bytes)
 818                                 ecclayout->eccpos[i] =
 819                                                 ecclayout->eccpos[i - 1] + 1;
 820                         else
 821                                 ecclayout->eccpos[i] =
 822                                                 ecclayout->eccpos[i - 1] + 2;
 823                 }
 824                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 825                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 826                                                 BADBLOCK_MARKER_LENGTH;
 827                 break;
 828 #else
 829                 printf("nand: error: CONFIG_BCH required for ECC\n");
 830                 return -EINVAL;
 831 #endif
 832
 833         case OMAP_ECC_BCH8_CODE_HW:
 834 #ifdef CONFIG_NAND_OMAP_ELM
 835                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW\n");
 836                 /* check ecc-scheme requirements before updating ecc info */
 837                 if ((14 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 838                         printf("nand: error: insufficient OOB: require=%d\n", (
 839                                 (14 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 840                         return -EINVAL;
 841                 }
 842                 /* intialize ELM for ECC error detection */
 843                 elm_init();
 844                 info->control           = NULL;
 845                 /* populate ecc specific fields */
 846                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 847                 nand->ecc.mode          = NAND_ECC_HW;
 848                 nand->ecc.strength      = 8;
 849                 nand->ecc.size          = SECTOR_BYTES;
 850                 nand->ecc.bytes         = 14;
 851                 nand->ecc.hwctl         = omap_enable_hwecc;
 852                 nand->ecc.correct       = omap_correct_data_bch;
 853                 nand->ecc.calculate     = omap_calculate_ecc;
 854                 nand->ecc.read_page     = omap_read_page_bch;
 855                 /* define ecc-layout */
 856                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 857                 for (i = 0; i < ecclayout->eccbytes; i++)
 858                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
 859                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 860                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 861                                                 BADBLOCK_MARKER_LENGTH;
 862                 break;
 863 #else
 864                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
 865                 return -EINVAL;
 866 #endif
 867
 868         case OMAP_ECC_BCH16_CODE_HW:
 869 #ifdef CONFIG_NAND_OMAP_ELM
 870                 debug("nand: using OMAP_ECC_BCH16_CODE_HW\n");
 871                 /* check ecc-scheme requirements before updating ecc info */
 872                 if ((26 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 873                         printf("nand: error: insufficient OOB: require=%d\n", (
 874                                 (26 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 875                         return -EINVAL;
 876                 }
 877                 /* intialize ELM for ECC error detection */
 878                 elm_init();
 879                 /* populate ecc specific fields */
 880                 nand->ecc.mode          = NAND_ECC_HW;
 881                 nand->ecc.size          = SECTOR_BYTES;
 882                 nand->ecc.bytes         = 26;
 883                 nand->ecc.strength      = 16;
 884                 nand->ecc.hwctl         = omap_enable_hwecc;
 885                 nand->ecc.correct       = omap_correct_data_bch;
 886                 nand->ecc.calculate     = omap_calculate_ecc;
 887                 nand->ecc.read_page     = omap_read_page_bch;
 888                 /* define ecc-layout */
 889                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 890                 for (i = 0; i < ecclayout->eccbytes; i++)
 891                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
 892                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 893                 ecclayout->oobfree[0].length = oobsize - nand->ecc.bytes -
 894                                                 BADBLOCK_MARKER_LENGTH;
 895                 break;
 896 #else
 897                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
 898                 return -EINVAL;
 899 #endif
 900         default:
 901                 debug("nand: error: ecc scheme not enabled or supported\n");
 902                 return -EINVAL;
 903         }
 904
 905         /* nand_scan_tail() sets ham1 sw ecc; hw ecc layout is set by driver */
 906         if (ecc_scheme != OMAP_ECC_HAM1_CODE_SW)
 907                 nand->ecc.layout = ecclayout;
 908
 909         info->ecc_scheme = ecc_scheme;
 910         return 0;
 911 }
 912
 913 #ifndef CONFIG_SPL_BUILD
 914 /*
 915  * omap_nand_switch_ecc - switch the ECC operation between different engines
 916  * (h/w and s/w) and different algorithms (hamming and BCHx)
 917  *
 918  * @hardware            - true if one of the HW engines should be used
 919  * @eccstrength         - the number of bits that could be corrected
 920  *                        (1 - hamming, 4 - BCH4, 8 - BCH8, 16 - BCH16)
 921  */
 922 int __maybe_unused omap_nand_switch_ecc(uint32_t hardware, uint32_t eccstrength)
 923 {
 924         struct nand_chip *nand;
 925         struct mtd_info *mtd = get_nand_dev_by_index(nand_curr_device);
 926         int err = 0;
 927
 928         if (!mtd) {
 929                 printf("nand: error: no NAND devices found\n");
 930                 return -ENODEV;
 931         }
 932
 933         nand = mtd_to_nand(mtd);
 934         nand->options |= NAND_OWN_BUFFERS;
 935         nand->options &= ~NAND_SUBPAGE_READ;
 936         /* Setup the ecc configurations again */
 937         if (hardware) {
 938                 if (eccstrength == 1) {
 939                         err = omap_select_ecc_scheme(nand,
 940                                         OMAP_ECC_HAM1_CODE_HW,
 941                                         mtd->writesize, mtd->oobsize);
 942                 } else if (eccstrength == 8) {
 943                         err = omap_select_ecc_scheme(nand,
 944                                         OMAP_ECC_BCH8_CODE_HW,
 945                                         mtd->writesize, mtd->oobsize);
 946                 } else if (eccstrength == 16) {
 947                         err = omap_select_ecc_scheme(nand,
 948                                         OMAP_ECC_BCH16_CODE_HW,
 949                                         mtd->writesize, mtd->oobsize);
 950                 } else {
 951                         printf("nand: error: unsupported ECC scheme\n");
 952                         return -EINVAL;
 953                 }
 954         } else {
 955                 if (eccstrength == 1) {
 956                         err = omap_select_ecc_scheme(nand,
 957                                         OMAP_ECC_HAM1_CODE_SW,
 958                                         mtd->writesize, mtd->oobsize);
 959                 } else if (eccstrength == 8) {
 960                         err = omap_select_ecc_scheme(nand,
 961                                         OMAP_ECC_BCH8_CODE_HW_DETECTION_SW,
 962                                         mtd->writesize, mtd->oobsize);
 963                 } else {
 964                         printf("nand: error: unsupported ECC scheme\n");
 965                         return -EINVAL;
 966                 }
 967         }
 968
 969         /* Update NAND handling after ECC mode switch */
 970         if (!err)
 971                 err = nand_scan_tail(mtd);
 972         return err;
 973 }
 974 #endif /* CONFIG_SPL_BUILD */
 975
 976 /*
 977  * Board-specific NAND initialization. The following members of the
 978  * argument are board-specific:
 979  * - IO_ADDR_R: address to read the 8 I/O lines of the flash device
 980  * - IO_ADDR_W: address to write the 8 I/O lines of the flash device
 981  * - cmd_ctrl: hardwarespecific function for accesing control-lines
 982  * - waitfunc: hardwarespecific function for accesing device ready/busy line
 983  * - ecc.hwctl: function to enable (reset) hardware ecc generator
 984  * - ecc.mode: mode of ecc, see defines
 985  * - chip_delay: chip dependent delay for transfering data from array to
 986  *   read regs (tR)
 987  * - options: various chip options. They can partly be set to inform
 988  *   nand_scan about special functionality. See the defines for further
 989  *   explanation
 990  */
 991 int board_nand_init(struct nand_chip *nand)
 992 {
 993         int32_t gpmc_config = 0;
 994         int cs = cs_next++;
 995         int err = 0;
 996         /*
 997          * xloader/Uboot's gpmc configuration would have configured GPMC for
 998          * nand type of memory. The following logic scans and latches on to the
 999          * first CS with NAND type memory.
1000          * TBD: need to make this logic generic to handle multiple CS NAND
1001          * devices.
1002          */
1003         while (cs < GPMC_MAX_CS) {
1004                 /* Check if NAND type is set */
1005                 if ((readl(&gpmc_cfg->cs[cs].config1) & 0xC00) == 0x800) {
1006                         /* Found it!! */
1007                         break;
1008                 }
1009                 cs++;
1010         }
1011         if (cs >= GPMC_MAX_CS) {
1012                 printf("nand: error: Unable to find NAND settings in "
1013                         "GPMC Configuration - quitting\n");
1014                 return -ENODEV;
1015         }
1016
1017         gpmc_config = readl(&gpmc_cfg->config);
1018         /* Disable Write protect */
1019         gpmc_config |= 0x10;
1020         writel(gpmc_config, &gpmc_cfg->config);
1021
1022         nand->IO_ADDR_R = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
1023         nand->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
1024         omap_nand_info[cs].control = NULL;
1025         omap_nand_info[cs].cs = cs;
1026         omap_nand_info[cs].ws = wscfg[cs];
1027         nand_set_controller_data(nand, &omap_nand_info[cs]);
1028         nand->cmd_ctrl  = omap_nand_hwcontrol;
1029         nand->options   |= NAND_NO_PADDING | NAND_CACHEPRG;
1030         nand->chip_delay = 100;
1031         nand->ecc.layout = &omap_ecclayout;
1032
1033         /* configure driver and controller based on NAND device bus-width */
1034         gpmc_config = readl(&gpmc_cfg->cs[cs].config1);
1035 #if defined(CONFIG_SYS_NAND_BUSWIDTH_16BIT)
1036         nand->options |= NAND_BUSWIDTH_16;
1037         writel(gpmc_config | (0x1 << 12), &gpmc_cfg->cs[cs].config1);
1038 #else
1039         nand->options &= ~NAND_BUSWIDTH_16;
1040         writel(gpmc_config & ~(0x1 << 12), &gpmc_cfg->cs[cs].config1);
1041 #endif
1042         /* select ECC scheme */
1043 #if defined(CONFIG_NAND_OMAP_ECCSCHEME)
1044         err = omap_select_ecc_scheme(nand, CONFIG_NAND_OMAP_ECCSCHEME,
1045                         CONFIG_SYS_NAND_PAGE_SIZE, CONFIG_SYS_NAND_OOBSIZE);
1046 #else
1047         /* pagesize and oobsize are not required to configure sw ecc-scheme */
1048         err = omap_select_ecc_scheme(nand, OMAP_ECC_HAM1_CODE_SW,
1049                         0, 0);
1050 #endif
1051         if (err)
1052                 return err;
1053
1054 #ifdef CONFIG_NAND_OMAP_GPMC_PREFETCH
1055         nand->read_buf = omap_nand_read_prefetch;
1056 #else
1057         if (nand->options & NAND_BUSWIDTH_16)
1058                 nand->read_buf = nand_read_buf16;
1059         else
1060                 nand->read_buf = nand_read_buf;
1061 #endif
1062
1063         nand->dev_ready = omap_dev_ready;
1064
1065         return 0;
1066 }