drivers/mtd/nand/omap_gpmc.c

   1 /*
   2  * (C) Copyright 2004-2008 Texas Instruments, <www.ti.com>
   3  * Rohit Choraria <rohitkc@ti.com>
   4  *
   5  * SPDX-License-Identifier:     GPL-2.0+
   6  */
   7
   8 #include <common.h>
   9 #include <asm/io.h>
  10 #include <asm/errno.h>
  11 #include <asm/arch/mem.h>
  12 #include <linux/mtd/omap_gpmc.h>
  13 #include <linux/mtd/nand_ecc.h>
  14 #include <linux/bch.h>
  15 #include <linux/compiler.h>
  16 #include <nand.h>
  17 #include <linux/mtd/omap_elm.h>
  18
  19 #define BADBLOCK_MARKER_LENGTH  2
  20 #define SECTOR_BYTES            512
  21 #define ECCCLEAR                (0x1 << 8)
  22 #define ECCRESULTREG1           (0x1 << 0)
  23 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
  24 #define BCH4_BIT_PAD            4
  25
  26 #ifdef CONFIG_BCH
  27 static u8  bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
  28                                 0x97, 0x79, 0xe5, 0x24, 0xb5};
  29 #endif
  30 static uint8_t cs_next;
  31 static __maybe_unused struct nand_ecclayout omap_ecclayout;
  32
  33 #if defined(CONFIG_NAND_OMAP_GPMC_WSCFG)
  34 static const int8_t wscfg[CONFIG_SYS_MAX_NAND_DEVICE] =
  35         { CONFIG_NAND_OMAP_GPMC_WSCFG };
  36 #else
  37 /* wscfg is preset to zero since its a static variable */
  38 static const int8_t wscfg[CONFIG_SYS_MAX_NAND_DEVICE];
  39 #endif
  40
  41 /*
  42  * Driver configurations
  43  */
  44 struct omap_nand_info {
  45         struct bch_control *control;
  46         enum omap_ecc ecc_scheme;
  47         uint8_t cs;
  48         uint8_t ws;             /* wait status pin (0,1) */
  49 };
  50
  51 /* We are wasting a bit of memory but al least we are safe */
  52 static struct omap_nand_info omap_nand_info[GPMC_MAX_CS];
  53
  54 /*
  55  * omap_nand_hwcontrol - Set the address pointers corretly for the
  56  *                      following address/data/command operation
  57  */
  58 static void omap_nand_hwcontrol(struct mtd_info *mtd, int32_t cmd,
  59                                 uint32_t ctrl)
  60 {
  61         register struct nand_chip *this = mtd->priv;
  62         struct omap_nand_info *info = this->priv;
  63         int cs = info->cs;
  64
  65         /*
  66          * Point the IO_ADDR to DATA and ADDRESS registers instead
  67          * of chip address
  68          */
  69         switch (ctrl) {
  70         case NAND_CTRL_CHANGE | NAND_CTRL_CLE:
  71                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
  72                 break;
  73         case NAND_CTRL_CHANGE | NAND_CTRL_ALE:
  74                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_adr;
  75                 break;
  76         case NAND_CTRL_CHANGE | NAND_NCE:
  77                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
  78                 break;
  79         }
  80
  81         if (cmd != NAND_CMD_NONE)
  82                 writeb(cmd, this->IO_ADDR_W);
  83 }
  84
  85 /* Check wait pin as dev ready indicator */
  86 static int omap_dev_ready(struct mtd_info *mtd)
  87 {
  88         register struct nand_chip *this = mtd->priv;
  89         struct omap_nand_info *info = this->priv;
  90         return gpmc_cfg->status & (1 << (8 + info->ws));
  91 }
  92
  93 /*
  94  * gen_true_ecc - This function will generate true ECC value, which
  95  * can be used when correcting data read from NAND flash memory core
  96  *
  97  * @ecc_buf:    buffer to store ecc code
  98  *
  99  * @return:     re-formatted ECC value
 100  */
 101 static uint32_t gen_true_ecc(uint8_t *ecc_buf)
 102 {
 103         return ecc_buf[0] | (ecc_buf[1] << 16) | ((ecc_buf[2] & 0xF0) << 20) |
 104                 ((ecc_buf[2] & 0x0F) << 8);
 105 }
 106
 107 /*
 108  * omap_correct_data - Compares the ecc read from nand spare area with ECC
 109  * registers values and corrects one bit error if it has occured
 110  * Further details can be had from OMAP TRM and the following selected links:
 111  * http://en.wikipedia.org/wiki/Hamming_code
 112  * http://www.cs.utexas.edu/users/plaxton/c/337/05f/slides/ErrorCorrection-4.pdf
 113  *
 114  * @mtd:                 MTD device structure
 115  * @dat:                 page data
 116  * @read_ecc:            ecc read from nand flash
 117  * @calc_ecc:            ecc read from ECC registers
 118  *
 119  * @return 0 if data is OK or corrected, else returns -1
 120  */
 121 static int __maybe_unused omap_correct_data(struct mtd_info *mtd, uint8_t *dat,
 122                                 uint8_t *read_ecc, uint8_t *calc_ecc)
 123 {
 124         uint32_t orig_ecc, new_ecc, res, hm;
 125         uint16_t parity_bits, byte;
 126         uint8_t bit;
 127
 128         /* Regenerate the orginal ECC */
 129         orig_ecc = gen_true_ecc(read_ecc);
 130         new_ecc = gen_true_ecc(calc_ecc);
 131         /* Get the XOR of real ecc */
 132         res = orig_ecc ^ new_ecc;
 133         if (res) {
 134                 /* Get the hamming width */
 135                 hm = hweight32(res);
 136                 /* Single bit errors can be corrected! */
 137                 if (hm == 12) {
 138                         /* Correctable data! */
 139                         parity_bits = res >> 16;
 140                         bit = (parity_bits & 0x7);
 141                         byte = (parity_bits >> 3) & 0x1FF;
 142                         /* Flip the bit to correct */
 143                         dat[byte] ^= (0x1 << bit);
 144                 } else if (hm == 1) {
 145                         printf("Error: Ecc is wrong\n");
 146                         /* ECC itself is corrupted */
 147                         return 2;
 148                 } else {
 149                         /*
 150                          * hm distance != parity pairs OR one, could mean 2 bit
 151                          * error OR potentially be on a blank page..
 152                          * orig_ecc: contains spare area data from nand flash.
 153                          * new_ecc: generated ecc while reading data area.
 154                          * Note: if the ecc = 0, all data bits from which it was
 155                          * generated are 0xFF.
 156                          * The 3 byte(24 bits) ecc is generated per 512byte
 157                          * chunk of a page. If orig_ecc(from spare area)
 158                          * is 0xFF && new_ecc(computed now from data area)=0x0,
 159                          * this means that data area is 0xFF and spare area is
 160                          * 0xFF. A sure sign of a erased page!
 161                          */
 162                         if ((orig_ecc == 0x0FFF0FFF) && (new_ecc == 0x00000000))
 163                                 return 0;
 164                         printf("Error: Bad compare! failed\n");
 165                         /* detected 2 bit error */
 166                         return -1;
 167                 }
 168         }
 169         return 0;
 170 }
 171
 172 /*
 173  * omap_enable_hwecc - configures GPMC as per ECC scheme before read/write
 174  * @mtd:        MTD device structure
 175  * @mode:       Read/Write mode
 176  */
 177 __maybe_unused
 178 static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode)
 179 {
 180         struct nand_chip        *nand   = mtd->priv;
 181         struct omap_nand_info   *info   = nand->priv;
 182         unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0;
 183         unsigned int ecc_algo = 0;
 184         unsigned int bch_type = 0;
 185         unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00;
 186         u32 ecc_size_config_val = 0;
 187         u32 ecc_config_val = 0;
 188         int cs = info->cs;
 189
 190         /* configure GPMC for specific ecc-scheme */
 191         switch (info->ecc_scheme) {
 192         case OMAP_ECC_HAM1_CODE_SW:
 193                 return;
 194         case OMAP_ECC_HAM1_CODE_HW:
 195                 ecc_algo = 0x0;
 196                 bch_type = 0x0;
 197                 bch_wrapmode = 0x00;
 198                 eccsize0 = 0xFF;
 199                 eccsize1 = 0xFF;
 200                 break;
 201         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 202         case OMAP_ECC_BCH8_CODE_HW:
 203                 ecc_algo = 0x1;
 204                 bch_type = 0x1;
 205                 if (mode == NAND_ECC_WRITE) {
 206                         bch_wrapmode = 0x01;
 207                         eccsize0 = 0;  /* extra bits in nibbles per sector */
 208                         eccsize1 = 28; /* OOB bits in nibbles per sector */
 209                 } else {
 210                         bch_wrapmode = 0x01;
 211                         eccsize0 = 26; /* ECC bits in nibbles per sector */
 212                         eccsize1 = 2;  /* non-ECC bits in nibbles per sector */
 213                 }
 214                 break;
 215         case OMAP_ECC_BCH16_CODE_HW:
 216                 ecc_algo = 0x1;
 217                 bch_type = 0x2;
 218                 if (mode == NAND_ECC_WRITE) {
 219                         bch_wrapmode = 0x01;
 220                         eccsize0 = 0;  /* extra bits in nibbles per sector */
 221                         eccsize1 = 52; /* OOB bits in nibbles per sector */
 222                 } else {
 223                         bch_wrapmode = 0x01;
 224                         eccsize0 = 52; /* ECC bits in nibbles per sector */
 225                         eccsize1 = 0;  /* non-ECC bits in nibbles per sector */
 226                 }
 227                 break;
 228         default:
 229                 return;
 230         }
 231         /* Clear ecc and enable bits */
 232         writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
 233         /* Configure ecc size for BCH */
 234         ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12);
 235         writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config);
 236
 237         /* Configure device details for BCH engine */
 238         ecc_config_val = ((ecc_algo << 16)      | /* HAM1 | BCHx */
 239                         (bch_type << 12)        | /* BCH4/BCH8/BCH16 */
 240                         (bch_wrapmode << 8)     | /* wrap mode */
 241                         (dev_width << 7)        | /* bus width */
 242                         (0x0 << 4)              | /* number of sectors */
 243                         (cs <<  1)              | /* ECC CS */
 244                         (0x1));                   /* enable ECC */
 245         writel(ecc_config_val, &gpmc_cfg->ecc_config);
 246 }
 247
 248 /*
 249  *  omap_calculate_ecc - Read ECC result
 250  *  @mtd:       MTD structure
 251  *  @dat:       unused
 252  *  @ecc_code:  ecc_code buffer
 253  *  Using noninverted ECC can be considered ugly since writing a blank
 254  *  page ie. padding will clear the ECC bytes. This is no problem as
 255  *  long nobody is trying to write data on the seemingly unused page.
 256  *  Reading an erased page will produce an ECC mismatch between
 257  *  generated and read ECC bytes that has to be dealt with separately.
 258  *  E.g. if page is 0xFF (fresh erased), and if HW ECC engine within GPMC
 259  *  is used, the result of read will be 0x0 while the ECC offsets of the
 260  *  spare area will be 0xFF which will result in an ECC mismatch.
 261  */
 262 static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat,
 263                                 uint8_t *ecc_code)
 264 {
 265         struct nand_chip *chip = mtd->priv;
 266         struct omap_nand_info *info = chip->priv;
 267         uint32_t *ptr, val = 0;
 268         int8_t i = 0, j;
 269
 270         switch (info->ecc_scheme) {
 271         case OMAP_ECC_HAM1_CODE_HW:
 272                 val = readl(&gpmc_cfg->ecc1_result);
 273                 ecc_code[0] = val & 0xFF;
 274                 ecc_code[1] = (val >> 16) & 0xFF;
 275                 ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
 276                 break;
 277 #ifdef CONFIG_BCH
 278         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 279 #endif
 280         case OMAP_ECC_BCH8_CODE_HW:
 281                 ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3];
 282                 val = readl(ptr);
 283                 ecc_code[i++] = (val >>  0) & 0xFF;
 284                 ptr--;
 285                 for (j = 0; j < 3; j++) {
 286                         val = readl(ptr);
 287                         ecc_code[i++] = (val >> 24) & 0xFF;
 288                         ecc_code[i++] = (val >> 16) & 0xFF;
 289                         ecc_code[i++] = (val >>  8) & 0xFF;
 290                         ecc_code[i++] = (val >>  0) & 0xFF;
 291                         ptr--;
 292                 }
 293                 break;
 294         case OMAP_ECC_BCH16_CODE_HW:
 295                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]);
 296                 ecc_code[i++] = (val >>  8) & 0xFF;
 297                 ecc_code[i++] = (val >>  0) & 0xFF;
 298                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]);
 299                 ecc_code[i++] = (val >> 24) & 0xFF;
 300                 ecc_code[i++] = (val >> 16) & 0xFF;
 301                 ecc_code[i++] = (val >>  8) & 0xFF;
 302                 ecc_code[i++] = (val >>  0) & 0xFF;
 303                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]);
 304                 ecc_code[i++] = (val >> 24) & 0xFF;
 305                 ecc_code[i++] = (val >> 16) & 0xFF;
 306                 ecc_code[i++] = (val >>  8) & 0xFF;
 307                 ecc_code[i++] = (val >>  0) & 0xFF;
 308                 for (j = 3; j >= 0; j--) {
 309                         val = readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j]
 310                                                                         );
 311                         ecc_code[i++] = (val >> 24) & 0xFF;
 312                         ecc_code[i++] = (val >> 16) & 0xFF;
 313                         ecc_code[i++] = (val >>  8) & 0xFF;
 314                         ecc_code[i++] = (val >>  0) & 0xFF;
 315                 }
 316                 break;
 317         default:
 318                 return -EINVAL;
 319         }
 320         /* ECC scheme specific syndrome customizations */
 321         switch (info->ecc_scheme) {
 322         case OMAP_ECC_HAM1_CODE_HW:
 323                 break;
 324 #ifdef CONFIG_BCH
 325         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 326
 327                 for (i = 0; i < chip->ecc.bytes; i++)
 328                         *(ecc_code + i) = *(ecc_code + i) ^
 329                                                 bch8_polynomial[i];
 330                 break;
 331 #endif
 332         case OMAP_ECC_BCH8_CODE_HW:
 333                 ecc_code[chip->ecc.bytes - 1] = 0x00;
 334                 break;
 335         case OMAP_ECC_BCH16_CODE_HW:
 336                 break;
 337         default:
 338                 return -EINVAL;
 339         }
 340         return 0;
 341 }
 342
 343 #ifdef CONFIG_NAND_OMAP_GPMC_PREFETCH
 344
 345 #define PREFETCH_CONFIG1_CS_SHIFT       24
 346 #define PREFETCH_FIFOTHRESHOLD_MAX      0x40
 347 #define PREFETCH_FIFOTHRESHOLD(val)     ((val) << 8)
 348 #define PREFETCH_STATUS_COUNT(val)      (val & 0x00003fff)
 349 #define PREFETCH_STATUS_FIFO_CNT(val)   ((val >> 24) & 0x7F)
 350 #define ENABLE_PREFETCH                 (1 << 7)
 351
 352 /**
 353  * omap_prefetch_enable - configures and starts prefetch transfer
 354  * @fifo_th: fifo threshold to be used for read/ write
 355  * @count: number of bytes to be transferred
 356  * @is_write: prefetch read(0) or write post(1) mode
 357  * @cs: chip select to use
 358  */
 359 static int omap_prefetch_enable(int fifo_th, unsigned int count, int is_write, int cs)
 360 {
 361         uint32_t val;
 362
 363         if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX)
 364                 return -EINVAL;
 365
 366         if (readl(&gpmc_cfg->prefetch_control))
 367                 return -EBUSY;
 368
 369         /* Set the amount of bytes to be prefetched */
 370         writel(count, &gpmc_cfg->prefetch_config2);
 371
 372         val = (cs << PREFETCH_CONFIG1_CS_SHIFT) | (is_write & 1) |
 373                 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH;
 374         writel(val, &gpmc_cfg->prefetch_config1);
 375
 376         /*  Start the prefetch engine */
 377         writel(1, &gpmc_cfg->prefetch_control);
 378
 379         return 0;
 380 }
 381
 382 /**
 383  * omap_prefetch_reset - disables and stops the prefetch engine
 384  */
 385 static void omap_prefetch_reset(void)
 386 {
 387         writel(0, &gpmc_cfg->prefetch_control);
 388         writel(0, &gpmc_cfg->prefetch_config1);
 389 }
 390
 391 static int __read_prefetch_aligned(struct nand_chip *chip, uint32_t *buf, int len)
 392 {
 393         int ret;
 394         uint32_t cnt;
 395         struct omap_nand_info *info = chip->priv;
 396
 397         ret = omap_prefetch_enable(PREFETCH_FIFOTHRESHOLD_MAX, len, 0, info->cs);
 398         if (ret < 0)
 399                 return ret;
 400
 401         do {
 402                 int i;
 403
 404                 cnt = readl(&gpmc_cfg->prefetch_status);
 405                 cnt = PREFETCH_STATUS_FIFO_CNT(cnt);
 406
 407                 for (i = 0; i < cnt / 4; i++) {
 408                         *buf++ = readl(CONFIG_SYS_NAND_BASE);
 409                         len -= 4;
 410                 }
 411         } while (len);
 412
 413         omap_prefetch_reset();
 414
 415         return 0;
 416 }
 417
 418 static inline void omap_nand_read(struct mtd_info *mtd, uint8_t *buf, int len)
 419 {
 420         struct nand_chip *chip = mtd->priv;
 421
 422         if (chip->options & NAND_BUSWIDTH_16)
 423                 nand_read_buf16(mtd, buf, len);
 424         else
 425                 nand_read_buf(mtd, buf, len);
 426 }
 427
 428 static void omap_nand_read_prefetch(struct mtd_info *mtd, uint8_t *buf, int len)
 429 {
 430         int ret;
 431         uint32_t head, tail;
 432         struct nand_chip *chip = mtd->priv;
 433
 434         /*
 435          * If the destination buffer is unaligned, start with reading
 436          * the overlap byte-wise.
 437          */
 438         head = ((uint32_t) buf) % 4;
 439         if (head) {
 440                 omap_nand_read(mtd, buf, head);
 441                 buf += head;
 442                 len -= head;
 443         }
 444
 445         /*
 446          * Only transfer multiples of 4 bytes in a pre-fetched fashion.
 447          * If there's a residue, care for it byte-wise afterwards.
 448          */
 449         tail = len % 4;
 450
 451         ret = __read_prefetch_aligned(chip, (uint32_t *)buf, len - tail);
 452         if (ret < 0) {
 453                 /* fallback in case the prefetch engine is busy */
 454                 omap_nand_read(mtd, buf, len);
 455         } else if (tail) {
 456                 buf += len - tail;
 457                 omap_nand_read(mtd, buf, tail);
 458         }
 459 }
 460 #endif /* CONFIG_NAND_OMAP_GPMC_PREFETCH */
 461
 462 #ifdef CONFIG_NAND_OMAP_ELM
 463 /*
 464  * omap_reverse_list - re-orders list elements in reverse order [internal]
 465  * @list:       pointer to start of list
 466  * @length:     length of list
 467 */
 468 static void omap_reverse_list(u8 *list, unsigned int length)
 469 {
 470         unsigned int i, j;
 471         unsigned int half_length = length / 2;
 472         u8 tmp;
 473         for (i = 0, j = length - 1; i < half_length; i++, j--) {
 474                 tmp = list[i];
 475                 list[i] = list[j];
 476                 list[j] = tmp;
 477         }
 478 }
 479
 480 /*
 481  * omap_correct_data_bch - Compares the ecc read from nand spare area
 482  * with ECC registers values and corrects one bit error if it has occured
 483  *
 484  * @mtd:        MTD device structure
 485  * @dat:        page data
 486  * @read_ecc:   ecc read from nand flash (ignored)
 487  * @calc_ecc:   ecc read from ECC registers
 488  *
 489  * @return 0 if data is OK or corrected, else returns -1
 490  */
 491 static int omap_correct_data_bch(struct mtd_info *mtd, uint8_t *dat,
 492                                 uint8_t *read_ecc, uint8_t *calc_ecc)
 493 {
 494         struct nand_chip *chip = mtd->priv;
 495         struct omap_nand_info *info = chip->priv;
 496         struct nand_ecc_ctrl *ecc = &chip->ecc;
 497         uint32_t error_count = 0, error_max;
 498         uint32_t error_loc[ELM_MAX_ERROR_COUNT];
 499         enum bch_level bch_type;
 500         uint32_t i, ecc_flag = 0;
 501         uint8_t count;
 502         uint32_t byte_pos, bit_pos;
 503         int err = 0;
 504
 505         /* check calculated ecc */
 506         for (i = 0; i < ecc->bytes && !ecc_flag; i++) {
 507                 if (calc_ecc[i] != 0x00)
 508                         ecc_flag = 1;
 509         }
 510         if (!ecc_flag)
 511                 return 0;
 512
 513         /* check for whether its a erased-page */
 514         ecc_flag = 0;
 515         for (i = 0; i < ecc->bytes && !ecc_flag; i++) {
 516                 if (read_ecc[i] != 0xff)
 517                         ecc_flag = 1;
 518         }
 519         if (!ecc_flag)
 520                 return 0;
 521
 522         /*
 523          * while reading ECC result we read it in big endian.
 524          * Hence while loading to ELM we have rotate to get the right endian.
 525          */
 526         switch (info->ecc_scheme) {
 527         case OMAP_ECC_BCH8_CODE_HW:
 528                 bch_type = BCH_8_BIT;
 529                 omap_reverse_list(calc_ecc, ecc->bytes - 1);
 530                 break;
 531         case OMAP_ECC_BCH16_CODE_HW:
 532                 bch_type = BCH_16_BIT;
 533                 omap_reverse_list(calc_ecc, ecc->bytes);
 534                 break;
 535         default:
 536                 return -EINVAL;
 537         }
 538         /* use elm module to check for errors */
 539         elm_config(bch_type);
 540         err = elm_check_error(calc_ecc, bch_type, &error_count, error_loc);
 541         if (err)
 542                 return err;
 543
 544         /* correct bch error */
 545         for (count = 0; count < error_count; count++) {
 546                 switch (info->ecc_scheme) {
 547                 case OMAP_ECC_BCH8_CODE_HW:
 548                         /* 14th byte in ECC is reserved to match ROM layout */
 549                         error_max = SECTOR_BYTES + (ecc->bytes - 1);
 550                         break;
 551                 case OMAP_ECC_BCH16_CODE_HW:
 552                         error_max = SECTOR_BYTES + ecc->bytes;
 553                         break;
 554                 default:
 555                         return -EINVAL;
 556                 }
 557                 byte_pos = error_max - (error_loc[count] / 8) - 1;
 558                 bit_pos  = error_loc[count] % 8;
 559                 if (byte_pos < SECTOR_BYTES) {
 560                         dat[byte_pos] ^= 1 << bit_pos;
 561                         printf("nand: bit-flip corrected @data=%d\n", byte_pos);
 562                 } else if (byte_pos < error_max) {
 563                         read_ecc[byte_pos - SECTOR_BYTES] ^= 1 << bit_pos;
 564                         printf("nand: bit-flip corrected @oob=%d\n", byte_pos -
 565                                                                 SECTOR_BYTES);
 566                 } else {
 567                         err = -EBADMSG;
 568                         printf("nand: error: invalid bit-flip location\n");
 569                 }
 570         }
 571         return (err) ? err : error_count;
 572 }
 573
 574 /**
 575  * omap_read_page_bch - hardware ecc based page read function
 576  * @mtd:        mtd info structure
 577  * @chip:       nand chip info structure
 578  * @buf:        buffer to store read data
 579  * @oob_required: caller expects OOB data read to chip->oob_poi
 580  * @page:       page number to read
 581  *
 582  */
 583 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 584                                 uint8_t *buf, int oob_required, int page)
 585 {
 586         int i, eccsize = chip->ecc.size;
 587         int eccbytes = chip->ecc.bytes;
 588         int eccsteps = chip->ecc.steps;
 589         uint8_t *p = buf;
 590         uint8_t *ecc_calc = chip->buffers->ecccalc;
 591         uint8_t *ecc_code = chip->buffers->ecccode;
 592         uint32_t *eccpos = chip->ecc.layout->eccpos;
 593         uint8_t *oob = chip->oob_poi;
 594         uint32_t data_pos;
 595         uint32_t oob_pos;
 596
 597         data_pos = 0;
 598         /* oob area start */
 599         oob_pos = (eccsize * eccsteps) + chip->ecc.layout->eccpos[0];
 600         oob += chip->ecc.layout->eccpos[0];
 601
 602         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize,
 603                                 oob += eccbytes) {
 604                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
 605                 /* read data */
 606                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1);
 607                 chip->read_buf(mtd, p, eccsize);
 608
 609                 /* read respective ecc from oob area */
 610                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
 611                 chip->read_buf(mtd, oob, eccbytes);
 612                 /* read syndrome */
 613                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
 614
 615                 data_pos += eccsize;
 616                 oob_pos += eccbytes;
 617         }
 618
 619         for (i = 0; i < chip->ecc.total; i++)
 620                 ecc_code[i] = chip->oob_poi[eccpos[i]];
 621
 622         eccsteps = chip->ecc.steps;
 623         p = buf;
 624
 625         for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 626                 int stat;
 627
 628                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
 629                 if (stat < 0)
 630                         mtd->ecc_stats.failed++;
 631                 else
 632                         mtd->ecc_stats.corrected += stat;
 633         }
 634         return 0;
 635 }
 636 #endif /* CONFIG_NAND_OMAP_ELM */
 637
 638 /*
 639  * OMAP3 BCH8 support (with BCH library)
 640  */
 641 #ifdef CONFIG_BCH
 642 /**
 643  * omap_correct_data_bch_sw - Decode received data and correct errors
 644  * @mtd: MTD device structure
 645  * @data: page data
 646  * @read_ecc: ecc read from nand flash
 647  * @calc_ecc: ecc read from HW ECC registers
 648  */
 649 static int omap_correct_data_bch_sw(struct mtd_info *mtd, u_char *data,
 650                                  u_char *read_ecc, u_char *calc_ecc)
 651 {
 652         int i, count;
 653         /* cannot correct more than 8 errors */
 654         unsigned int errloc[8];
 655         struct nand_chip *chip = mtd->priv;
 656         struct omap_nand_info *info = chip->priv;
 657
 658         count = decode_bch(info->control, NULL, 512, read_ecc, calc_ecc,
 659                                                         NULL, errloc);
 660         if (count > 0) {
 661                 /* correct errors */
 662                 for (i = 0; i < count; i++) {
 663                         /* correct data only, not ecc bytes */
 664                         if (errloc[i] < 8*512)
 665                                 data[errloc[i]/8] ^= 1 << (errloc[i] & 7);
 666                         printf("corrected bitflip %u\n", errloc[i]);
 667 #ifdef DEBUG
 668                         puts("read_ecc: ");
 669                         /*
 670                          * BCH8 have 13 bytes of ECC; BCH4 needs adoption
 671                          * here!
 672                          */
 673                         for (i = 0; i < 13; i++)
 674                                 printf("%02x ", read_ecc[i]);
 675                         puts("\n");
 676                         puts("calc_ecc: ");
 677                         for (i = 0; i < 13; i++)
 678                                 printf("%02x ", calc_ecc[i]);
 679                         puts("\n");
 680 #endif
 681                 }
 682         } else if (count < 0) {
 683                 puts("ecc unrecoverable error\n");
 684         }
 685         return count;
 686 }
 687
 688 /**
 689  * omap_free_bch - Release BCH ecc resources
 690  * @mtd: MTD device structure
 691  */
 692 static void __maybe_unused omap_free_bch(struct mtd_info *mtd)
 693 {
 694         struct nand_chip *chip = mtd->priv;
 695         struct omap_nand_info *info = chip->priv;
 696
 697         if (info->control) {
 698                 free_bch(info->control);
 699                 info->control = NULL;
 700         }
 701 }
 702 #endif /* CONFIG_BCH */
 703
 704 /**
 705  * omap_select_ecc_scheme - configures driver for particular ecc-scheme
 706  * @nand: NAND chip device structure
 707  * @ecc_scheme: ecc scheme to configure
 708  * @pagesize: number of main-area bytes per page of NAND device
 709  * @oobsize: number of OOB/spare bytes per page of NAND device
 710  */
 711 static int omap_select_ecc_scheme(struct nand_chip *nand,
 712         enum omap_ecc ecc_scheme, unsigned int pagesize, unsigned int oobsize) {
 713         struct omap_nand_info   *info           = nand->priv;
 714         struct nand_ecclayout   *ecclayout      = &omap_ecclayout;
 715         int eccsteps = pagesize / SECTOR_BYTES;
 716         int i;
 717
 718         switch (ecc_scheme) {
 719         case OMAP_ECC_HAM1_CODE_SW:
 720                 debug("nand: selected OMAP_ECC_HAM1_CODE_SW\n");
 721                 /* For this ecc-scheme, ecc.bytes, ecc.layout, ... are
 722                  * initialized in nand_scan_tail(), so just set ecc.mode */
 723                 info->control           = NULL;
 724                 nand->ecc.mode          = NAND_ECC_SOFT;
 725                 nand->ecc.layout        = NULL;
 726                 nand->ecc.size          = 0;
 727                 break;
 728
 729         case OMAP_ECC_HAM1_CODE_HW:
 730                 debug("nand: selected OMAP_ECC_HAM1_CODE_HW\n");
 731                 /* check ecc-scheme requirements before updating ecc info */
 732                 if ((3 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 733                         printf("nand: error: insufficient OOB: require=%d\n", (
 734                                 (3 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 735                         return -EINVAL;
 736                 }
 737                 info->control           = NULL;
 738                 /* populate ecc specific fields */
 739                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 740                 nand->ecc.mode          = NAND_ECC_HW;
 741                 nand->ecc.strength      = 1;
 742                 nand->ecc.size          = SECTOR_BYTES;
 743                 nand->ecc.bytes         = 3;
 744                 nand->ecc.hwctl         = omap_enable_hwecc;
 745                 nand->ecc.correct       = omap_correct_data;
 746                 nand->ecc.calculate     = omap_calculate_ecc;
 747                 /* define ecc-layout */
 748                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 749                 for (i = 0; i < ecclayout->eccbytes; i++) {
 750                         if (nand->options & NAND_BUSWIDTH_16)
 751                                 ecclayout->eccpos[i] = i + 2;
 752                         else
 753                                 ecclayout->eccpos[i] = i + 1;
 754                 }
 755                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 756                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 757                                                 BADBLOCK_MARKER_LENGTH;
 758                 break;
 759
 760         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
 761 #ifdef CONFIG_BCH
 762                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
 763                 /* check ecc-scheme requirements before updating ecc info */
 764                 if ((13 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 765                         printf("nand: error: insufficient OOB: require=%d\n", (
 766                                 (13 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 767                         return -EINVAL;
 768                 }
 769                 /* check if BCH S/W library can be used for error detection */
 770                 info->control = init_bch(13, 8, 0x201b);
 771                 if (!info->control) {
 772                         printf("nand: error: could not init_bch()\n");
 773                         return -ENODEV;
 774                 }
 775                 /* populate ecc specific fields */
 776                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 777                 nand->ecc.mode          = NAND_ECC_HW;
 778                 nand->ecc.strength      = 8;
 779                 nand->ecc.size          = SECTOR_BYTES;
 780                 nand->ecc.bytes         = 13;
 781                 nand->ecc.hwctl         = omap_enable_hwecc;
 782                 nand->ecc.correct       = omap_correct_data_bch_sw;
 783                 nand->ecc.calculate     = omap_calculate_ecc;
 784                 /* define ecc-layout */
 785                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 786                 ecclayout->eccpos[0]    = BADBLOCK_MARKER_LENGTH;
 787                 for (i = 1; i < ecclayout->eccbytes; i++) {
 788                         if (i % nand->ecc.bytes)
 789                                 ecclayout->eccpos[i] =
 790                                                 ecclayout->eccpos[i - 1] + 1;
 791                         else
 792                                 ecclayout->eccpos[i] =
 793                                                 ecclayout->eccpos[i - 1] + 2;
 794                 }
 795                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 796                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 797                                                 BADBLOCK_MARKER_LENGTH;
 798                 break;
 799 #else
 800                 printf("nand: error: CONFIG_BCH required for ECC\n");
 801                 return -EINVAL;
 802 #endif
 803
 804         case OMAP_ECC_BCH8_CODE_HW:
 805 #ifdef CONFIG_NAND_OMAP_ELM
 806                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW\n");
 807                 /* check ecc-scheme requirements before updating ecc info */
 808                 if ((14 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 809                         printf("nand: error: insufficient OOB: require=%d\n", (
 810                                 (14 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 811                         return -EINVAL;
 812                 }
 813                 /* intialize ELM for ECC error detection */
 814                 elm_init();
 815                 info->control           = NULL;
 816                 /* populate ecc specific fields */
 817                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
 818                 nand->ecc.mode          = NAND_ECC_HW;
 819                 nand->ecc.strength      = 8;
 820                 nand->ecc.size          = SECTOR_BYTES;
 821                 nand->ecc.bytes         = 14;
 822                 nand->ecc.hwctl         = omap_enable_hwecc;
 823                 nand->ecc.correct       = omap_correct_data_bch;
 824                 nand->ecc.calculate     = omap_calculate_ecc;
 825                 nand->ecc.read_page     = omap_read_page_bch;
 826                 /* define ecc-layout */
 827                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 828                 for (i = 0; i < ecclayout->eccbytes; i++)
 829                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
 830                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 831                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
 832                                                 BADBLOCK_MARKER_LENGTH;
 833                 break;
 834 #else
 835                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
 836                 return -EINVAL;
 837 #endif
 838
 839         case OMAP_ECC_BCH16_CODE_HW:
 840 #ifdef CONFIG_NAND_OMAP_ELM
 841                 debug("nand: using OMAP_ECC_BCH16_CODE_HW\n");
 842                 /* check ecc-scheme requirements before updating ecc info */
 843                 if ((26 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
 844                         printf("nand: error: insufficient OOB: require=%d\n", (
 845                                 (26 * eccsteps) + BADBLOCK_MARKER_LENGTH));
 846                         return -EINVAL;
 847                 }
 848                 /* intialize ELM for ECC error detection */
 849                 elm_init();
 850                 /* populate ecc specific fields */
 851                 nand->ecc.mode          = NAND_ECC_HW;
 852                 nand->ecc.size          = SECTOR_BYTES;
 853                 nand->ecc.bytes         = 26;
 854                 nand->ecc.strength      = 16;
 855                 nand->ecc.hwctl         = omap_enable_hwecc;
 856                 nand->ecc.correct       = omap_correct_data_bch;
 857                 nand->ecc.calculate     = omap_calculate_ecc;
 858                 nand->ecc.read_page     = omap_read_page_bch;
 859                 /* define ecc-layout */
 860                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
 861                 for (i = 0; i < ecclayout->eccbytes; i++)
 862                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
 863                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
 864                 ecclayout->oobfree[0].length = oobsize - nand->ecc.bytes -
 865                                                 BADBLOCK_MARKER_LENGTH;
 866                 break;
 867 #else
 868                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
 869                 return -EINVAL;
 870 #endif
 871         default:
 872                 debug("nand: error: ecc scheme not enabled or supported\n");
 873                 return -EINVAL;
 874         }
 875
 876         /* nand_scan_tail() sets ham1 sw ecc; hw ecc layout is set by driver */
 877         if (ecc_scheme != OMAP_ECC_HAM1_CODE_SW)
 878                 nand->ecc.layout = ecclayout;
 879
 880         info->ecc_scheme = ecc_scheme;
 881         return 0;
 882 }
 883
 884 #ifndef CONFIG_SPL_BUILD
 885 /*
 886  * omap_nand_switch_ecc - switch the ECC operation between different engines
 887  * (h/w and s/w) and different algorithms (hamming and BCHx)
 888  *
 889  * @hardware            - true if one of the HW engines should be used
 890  * @eccstrength         - the number of bits that could be corrected
 891  *                        (1 - hamming, 4 - BCH4, 8 - BCH8, 16 - BCH16)
 892  */
 893 int __maybe_unused omap_nand_switch_ecc(uint32_t hardware, uint32_t eccstrength)
 894 {
 895         struct nand_chip *nand;
 896         struct mtd_info *mtd;
 897         int err = 0;
 898
 899         if (nand_curr_device < 0 ||
 900             nand_curr_device >= CONFIG_SYS_MAX_NAND_DEVICE ||
 901             !nand_info[nand_curr_device].name) {
 902                 printf("nand: error: no NAND devices found\n");
 903                 return -ENODEV;
 904         }
 905
 906         mtd = &nand_info[nand_curr_device];
 907         nand = mtd->priv;
 908         nand->options |= NAND_OWN_BUFFERS;
 909         nand->options &= ~NAND_SUBPAGE_READ;
 910         /* Setup the ecc configurations again */
 911         if (hardware) {
 912                 if (eccstrength == 1) {
 913                         err = omap_select_ecc_scheme(nand,
 914                                         OMAP_ECC_HAM1_CODE_HW,
 915                                         mtd->writesize, mtd->oobsize);
 916                 } else if (eccstrength == 8) {
 917                         err = omap_select_ecc_scheme(nand,
 918                                         OMAP_ECC_BCH8_CODE_HW,
 919                                         mtd->writesize, mtd->oobsize);
 920                 } else {
 921                         printf("nand: error: unsupported ECC scheme\n");
 922                         return -EINVAL;
 923                 }
 924         } else {
 925                 if (eccstrength == 1) {
 926                         err = omap_select_ecc_scheme(nand,
 927                                         OMAP_ECC_HAM1_CODE_SW,
 928                                         mtd->writesize, mtd->oobsize);
 929                 } else if (eccstrength == 8) {
 930                         err = omap_select_ecc_scheme(nand,
 931                                         OMAP_ECC_BCH8_CODE_HW_DETECTION_SW,
 932                                         mtd->writesize, mtd->oobsize);
 933                 } else {
 934                         printf("nand: error: unsupported ECC scheme\n");
 935                         return -EINVAL;
 936                 }
 937         }
 938
 939         /* Update NAND handling after ECC mode switch */
 940         if (!err)
 941                 err = nand_scan_tail(mtd);
 942         return err;
 943 }
 944 #endif /* CONFIG_SPL_BUILD */
 945
 946 /*
 947  * Board-specific NAND initialization. The following members of the
 948  * argument are board-specific:
 949  * - IO_ADDR_R: address to read the 8 I/O lines of the flash device
 950  * - IO_ADDR_W: address to write the 8 I/O lines of the flash device
 951  * - cmd_ctrl: hardwarespecific function for accesing control-lines
 952  * - waitfunc: hardwarespecific function for accesing device ready/busy line
 953  * - ecc.hwctl: function to enable (reset) hardware ecc generator
 954  * - ecc.mode: mode of ecc, see defines
 955  * - chip_delay: chip dependent delay for transfering data from array to
 956  *   read regs (tR)
 957  * - options: various chip options. They can partly be set to inform
 958  *   nand_scan about special functionality. See the defines for further
 959  *   explanation
 960  */
 961 int board_nand_init(struct nand_chip *nand)
 962 {
 963         int32_t gpmc_config = 0;
 964         int cs = cs_next++;
 965         int err = 0;
 966         /*
 967          * xloader/Uboot's gpmc configuration would have configured GPMC for
 968          * nand type of memory. The following logic scans and latches on to the
 969          * first CS with NAND type memory.
 970          * TBD: need to make this logic generic to handle multiple CS NAND
 971          * devices.
 972          */
 973         while (cs < GPMC_MAX_CS) {
 974                 /* Check if NAND type is set */
 975                 if ((readl(&gpmc_cfg->cs[cs].config1) & 0xC00) == 0x800) {
 976                         /* Found it!! */
 977                         break;
 978                 }
 979                 cs++;
 980         }
 981         if (cs >= GPMC_MAX_CS) {
 982                 printf("nand: error: Unable to find NAND settings in "
 983                         "GPMC Configuration - quitting\n");
 984                 return -ENODEV;
 985         }
 986
 987         gpmc_config = readl(&gpmc_cfg->config);
 988         /* Disable Write protect */
 989         gpmc_config |= 0x10;
 990         writel(gpmc_config, &gpmc_cfg->config);
 991
 992         nand->IO_ADDR_R = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
 993         nand->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
 994         omap_nand_info[cs].control = NULL;
 995         omap_nand_info[cs].cs = cs;
 996         omap_nand_info[cs].ws = wscfg[cs];
 997         nand->priv      = &omap_nand_info[cs];
 998         nand->cmd_ctrl  = omap_nand_hwcontrol;
 999         nand->options   |= NAND_NO_PADDING | NAND_CACHEPRG;
1000         nand->chip_delay = 100;
1001         nand->ecc.layout = &omap_ecclayout;
1002
1003         /* configure driver and controller based on NAND device bus-width */
1004         gpmc_config = readl(&gpmc_cfg->cs[cs].config1);
1005 #if defined(CONFIG_SYS_NAND_BUSWIDTH_16BIT)
1006         nand->options |= NAND_BUSWIDTH_16;
1007         writel(gpmc_config | (0x1 << 12), &gpmc_cfg->cs[cs].config1);
1008 #else
1009         nand->options &= ~NAND_BUSWIDTH_16;
1010         writel(gpmc_config & ~(0x1 << 12), &gpmc_cfg->cs[cs].config1);
1011 #endif
1012         /* select ECC scheme */
1013 #if defined(CONFIG_NAND_OMAP_ECCSCHEME)
1014         err = omap_select_ecc_scheme(nand, CONFIG_NAND_OMAP_ECCSCHEME,
1015                         CONFIG_SYS_NAND_PAGE_SIZE, CONFIG_SYS_NAND_OOBSIZE);
1016 #else
1017         /* pagesize and oobsize are not required to configure sw ecc-scheme */
1018         err = omap_select_ecc_scheme(nand, OMAP_ECC_HAM1_CODE_SW,
1019                         0, 0);
1020 #endif
1021         if (err)
1022                 return err;
1023
1024 #ifdef CONFIG_NAND_OMAP_GPMC_PREFETCH
1025         nand->read_buf = omap_nand_read_prefetch;
1026 #else
1027         if (nand->options & NAND_BUSWIDTH_16)
1028                 nand->read_buf = nand_read_buf16;
1029         else
1030                 nand->read_buf = nand_read_buf;
1031 #endif
1032
1033         nand->dev_ready = omap_dev_ready;
1034
1035         return 0;
1036 }