drivers/mtd/nand/raw/octeontx_nand.c

   1 // SPDX-License-Identifier:    GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Marvell International Ltd.
   4  */
   5
   6 #include <dm.h>
   7 #include <dm/device-internal.h>
   8 #include <dm/devres.h>
   9 #include <dm/of_access.h>
  10 #include <malloc.h>
  11 #include <memalign.h>
  12 #include <nand.h>
  13 #include <pci.h>
  14 #include <time.h>
  15 #include <linux/bitfield.h>
  16 #include <linux/ctype.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/delay.h>
  19 #include <linux/errno.h>
  20 #include <linux/err.h>
  21 #include <linux/ioport.h>
  22 #include <linux/libfdt.h>
  23 #include <linux/mtd/mtd.h>
  24 #include <linux/mtd/nand_bch.h>
  25 #include <linux/mtd/nand_ecc.h>
  26 #include <asm/io.h>
  27 #include <asm/types.h>
  28 #include <asm/dma-mapping.h>
  29 #include <asm/arch/clock.h>
  30 #include "octeontx_bch.h"
  31
  32 #ifdef DEBUG
  33 # undef CONFIG_LOGLEVEL
  34 # define CONFIG_LOGLEVEL 8
  35 #endif
  36
  37 /*
  38  * The NDF_CMD queue takes commands between 16 - 128 bit.
  39  * All commands must be 16 bit aligned and are little endian.
  40  * WAIT_STATUS commands must be 64 bit aligned.
  41  * Commands are selected by the 4 bit opcode.
  42  *
  43  * Available Commands:
  44  *
  45  * 16 Bit:
  46  *   NOP
  47  *   WAIT
  48  *   BUS_ACQ, BUS_REL
  49  *   CHIP_EN, CHIP_DIS
  50  *
  51  * 32 Bit:
  52  *   CLE_CMD
  53  *   RD_CMD, RD_EDO_CMD
  54  *   WR_CMD
  55  *
  56  * 64 Bit:
  57  *   SET_TM_PAR
  58  *
  59  * 96 Bit:
  60  *   ALE_CMD
  61  *
  62  * 128 Bit:
  63  *   WAIT_STATUS, WAIT_STATUS_ALE
  64  */
  65
  66 /* NDF Register offsets */
  67 #define NDF_CMD                 0x0
  68 #define NDF_MISC                0x8
  69 #define NDF_ECC_CNT             0x10
  70 #define NDF_DRBELL              0x30
  71 #define NDF_ST_REG              0x38    /* status */
  72 #define NDF_INT                 0x40
  73 #define NDF_INT_W1S             0x48
  74 #define NDF_DMA_CFG             0x50
  75 #define NDF_DMA_ADR             0x58
  76 #define NDF_INT_ENA_W1C         0x60
  77 #define NDF_INT_ENA_W1S         0x68
  78
  79 /* NDF command opcodes */
  80 #define NDF_OP_NOP              0x0
  81 #define NDF_OP_SET_TM_PAR       0x1
  82 #define NDF_OP_WAIT             0x2
  83 #define NDF_OP_CHIP_EN_DIS      0x3
  84 #define NDF_OP_CLE_CMD          0x4
  85 #define NDF_OP_ALE_CMD          0x5
  86 #define NDF_OP_WR_CMD           0x8
  87 #define NDF_OP_RD_CMD           0x9
  88 #define NDF_OP_RD_EDO_CMD       0xa
  89 #define NDF_OP_WAIT_STATUS      0xb     /* same opcode for WAIT_STATUS_ALE */
  90 #define NDF_OP_BUS_ACQ_REL      0xf
  91
  92 #define NDF_BUS_ACQUIRE         1
  93 #define NDF_BUS_RELEASE         0
  94
  95 #define DBGX_EDSCR(X)           (0x87A008000088 + (X) * 0x80000)
  96
  97 struct ndf_nop_cmd {
  98         u16 opcode:     4;
  99         u16 nop:        12;
 100 };
 101
 102 struct ndf_wait_cmd {
 103         u16 opcode:4;
 104         u16 r_b:1;              /* wait for one cycle or PBUS_WAIT deassert */
 105         u16:3;
 106         u16 wlen:3;             /* timing parameter select */
 107         u16:5;
 108 };
 109
 110 struct ndf_bus_cmd {
 111         u16 opcode:4;
 112         u16 direction:4;        /* 1 = acquire, 0 = release */
 113         u16:8;
 114 };
 115
 116 struct ndf_chip_cmd {
 117         u16 opcode:4;
 118         u16 chip:3;             /* select chip, 0 = disable */
 119         u16 enable:1;           /* 1 = enable, 0 = disable */
 120         u16 bus_width:2;        /* 10 = 16 bit, 01 = 8 bit */
 121         u16:6;
 122 };
 123
 124 struct ndf_cle_cmd {
 125         u32 opcode:4;
 126         u32:4;
 127         u32 cmd_data:8;         /* command sent to the PBUS AD pins */
 128         u32 clen1:3;            /* time between PBUS CLE and WE asserts */
 129         u32 clen2:3;            /* time WE remains asserted */
 130         u32 clen3:3;            /* time between WE deassert and CLE */
 131         u32:7;
 132 };
 133
 134 /* RD_EDO_CMD uses the same layout as RD_CMD */
 135 struct ndf_rd_cmd {
 136         u32 opcode:4;
 137         u32 data:16;            /* data bytes */
 138         u32 rlen1:3;
 139         u32 rlen2:3;
 140         u32 rlen3:3;
 141         u32 rlen4:3;
 142 };
 143
 144 struct ndf_wr_cmd {
 145         u32 opcode:4;
 146         u32 data:16;            /* data bytes */
 147         u32:4;
 148         u32 wlen1:3;
 149         u32 wlen2:3;
 150         u32:3;
 151 };
 152
 153 struct ndf_set_tm_par_cmd {
 154         u64 opcode:4;
 155         u64 tim_mult:4; /* multiplier for the seven parameters */
 156         u64 tm_par1:8;  /* --> Following are the 7 timing parameters that */
 157         u64 tm_par2:8;  /*     specify the number of coprocessor cycles.  */
 158         u64 tm_par3:8;  /*     A value of zero means one cycle.           */
 159         u64 tm_par4:8;  /*     All values are scaled by tim_mult          */
 160         u64 tm_par5:8;  /*     using tim_par * (2 ^ tim_mult).            */
 161         u64 tm_par6:8;
 162         u64 tm_par7:8;
 163 };
 164
 165 struct ndf_ale_cmd {
 166         u32 opcode:4;
 167         u32:4;
 168         u32 adr_byte_num:4;     /* number of address bytes to be sent */
 169         u32:4;
 170         u32 alen1:3;
 171         u32 alen2:3;
 172         u32 alen3:3;
 173         u32 alen4:3;
 174         u32:4;
 175         u8 adr_byt1;
 176         u8 adr_byt2;
 177         u8 adr_byt3;
 178         u8 adr_byt4;
 179         u8 adr_byt5;
 180         u8 adr_byt6;
 181         u8 adr_byt7;
 182         u8 adr_byt8;
 183 };
 184
 185 struct ndf_wait_status_cmd {
 186         u32 opcode:4;
 187         u32:4;
 188         u32 data:8;             /** data */
 189         u32 clen1:3;
 190         u32 clen2:3;
 191         u32 clen3:3;
 192         u32:8;
 193         /** set to 5 to select WAIT_STATUS_ALE command */
 194         u32 ale_ind:8;
 195         /** ALE only: number of address bytes to be sent */
 196         u32 adr_byte_num:4;
 197         u32:4;
 198         u32 alen1:3;    /* ALE only */
 199         u32 alen2:3;    /* ALE only */
 200         u32 alen3:3;    /* ALE only */
 201         u32 alen4:3;    /* ALE only */
 202         u32:4;
 203         u8 adr_byt[4];          /* ALE only */
 204         u32 nine:4;     /* set to 9 */
 205         u32 and_mask:8;
 206         u32 comp_byte:8;
 207         u32 rlen1:3;
 208         u32 rlen2:3;
 209         u32 rlen3:3;
 210         u32 rlen4:3;
 211 };
 212
 213 union ndf_cmd {
 214         u64 val[2];
 215         union {
 216                 struct ndf_nop_cmd              nop;
 217                 struct ndf_wait_cmd             wait;
 218                 struct ndf_bus_cmd              bus_acq_rel;
 219                 struct ndf_chip_cmd             chip_en_dis;
 220                 struct ndf_cle_cmd              cle_cmd;
 221                 struct ndf_rd_cmd               rd_cmd;
 222                 struct ndf_wr_cmd               wr_cmd;
 223                 struct ndf_set_tm_par_cmd       set_tm_par;
 224                 struct ndf_ale_cmd              ale_cmd;
 225                 struct ndf_wait_status_cmd      wait_status;
 226         } u;
 227 };
 228
 229 /** Disable multi-bit error hangs */
 230 #define NDF_MISC_MB_DIS         BIT_ULL(27)
 231 /** High watermark for NBR FIFO or load/store operations */
 232 #define NDF_MISC_NBR_HWM        GENMASK_ULL(26, 24)
 233 /** Wait input filter count */
 234 #define NDF_MISC_WAIT_CNT       GENMASK_ULL(23, 18)
 235 /** Unfilled NFD_CMD queue bytes */
 236 #define NDF_MISC_FR_BYTE        GENMASK_ULL(17, 7)
 237 /** Set by HW when it reads the last 8 bytes of NDF_CMD */
 238 #define NDF_MISC_RD_DONE        BIT_ULL(6)
 239 /** Set by HW when it reads. SW read of NDF_CMD clears it */
 240 #define NDF_MISC_RD_VAL         BIT_ULL(5)
 241 /** Let HW read NDF_CMD queue. Cleared on SW NDF_CMD write */
 242 #define NDF_MISC_RD_CMD         BIT_ULL(4)
 243 /** Boot disable */
 244 #define NDF_MISC_BT_DIS         BIT_ULL(2)
 245 /** Stop command execution after completing command queue */
 246 #define NDF_MISC_EX_DIS         BIT_ULL(1)
 247 /** Reset fifo */
 248 #define NDF_MISC_RST_FF         BIT_ULL(0)
 249
 250 /** DMA engine enable */
 251 #define NDF_DMA_CFG_EN          BIT_ULL(63)
 252 /** Read or write */
 253 #define NDF_DMA_CFG_RW          BIT_ULL(62)
 254 /** Terminates DMA and clears enable bit */
 255 #define NDF_DMA_CFG_CLR         BIT_ULL(61)
 256 /** 32-bit swap enable */
 257 #define NDF_DMA_CFG_SWAP32      BIT_ULL(59)
 258 /** 16-bit swap enable */
 259 #define NDF_DMA_CFG_SWAP16      BIT_ULL(58)
 260 /** 8-bit swap enable */
 261 #define NDF_DMA_CFG_SWAP8       BIT_ULL(57)
 262 /** Endian mode */
 263 #define NDF_DMA_CFG_CMD_BE      BIT_ULL(56)
 264 /** Number of 64 bit transfers */
 265 #define NDF_DMA_CFG_SIZE        GENMASK_ULL(55, 36)
 266
 267 /** Command execution status idle */
 268 #define NDF_ST_REG_EXE_IDLE     BIT_ULL(15)
 269 /** Command execution SM states */
 270 #define NDF_ST_REG_EXE_SM       GENMASK_ULL(14, 11)
 271 /** DMA and load SM states */
 272 #define NDF_ST_REG_BT_SM        GENMASK_ULL(10, 7)
 273 /** Queue read-back SM bad state */
 274 #define NDF_ST_REG_RD_FF_BAD    BIT_ULL(6)
 275 /** Queue read-back SM states */
 276 #define NDF_ST_REG_RD_FF        GENMASK_ULL(5, 4)
 277 /** Main SM is in a bad state */
 278 #define NDF_ST_REG_MAIN_BAD     BIT_ULL(3)
 279 /** Main SM states */
 280 #define NDF_ST_REG_MAIN_SM      GENMASK_ULL(2, 0)
 281
 282 #define MAX_NAND_NAME_LEN       64
 283 #if (defined(NAND_MAX_PAGESIZE) && (NAND_MAX_PAGESIZE > 4096)) ||       \
 284         !defined(NAND_MAX_PAGESIZE)
 285 # undef NAND_MAX_PAGESIZE
 286 # define NAND_MAX_PAGESIZE      4096
 287 #endif
 288 #if (defined(NAND_MAX_OOBSIZE) && (NAND_MAX_OOBSIZE > 256)) ||          \
 289         !defined(NAND_MAX_OOBSIZE)
 290 # undef NAND_MAX_OOBSIZE
 291 # define NAND_MAX_OOBSIZE       256
 292 #endif
 293
 294 #define OCTEONTX_NAND_DRIVER_NAME       "octeontx_nand"
 295
 296 #define NDF_TIMEOUT             1000    /** Timeout in ms */
 297 #define USEC_PER_SEC            1000000 /** Linux compatibility */
 298 #ifndef NAND_MAX_CHIPS
 299 # define NAND_MAX_CHIPS         8       /** Linux compatibility */
 300 #endif
 301
 302 struct octeontx_nand_chip {
 303         struct list_head node;
 304         struct nand_chip nand;
 305         struct ndf_set_tm_par_cmd timings;
 306         int cs;
 307         int selected_page;
 308         int iface_mode;
 309         int row_bytes;
 310         int col_bytes;
 311         bool oob_only;
 312         bool iface_set;
 313 };
 314
 315 struct octeontx_nand_buf {
 316         u8 *dmabuf;
 317         dma_addr_t dmaaddr;
 318         int dmabuflen;
 319         int data_len;
 320         int data_index;
 321 };
 322
 323 /** NAND flash controller (NDF) related information */
 324 struct octeontx_nfc {
 325         struct nand_hw_control controller;
 326         struct udevice *dev;
 327         void __iomem *base;
 328         struct list_head chips;
 329         int selected_chip;      /* Currently selected NAND chip number */
 330
 331         /*
 332          * Status is separate from octeontx_nand_buf because
 333          * it can be used in parallel and during init.
 334          */
 335         u8 *stat;
 336         dma_addr_t stat_addr;
 337         bool use_status;
 338
 339         struct octeontx_nand_buf buf;
 340         union bch_resp *bch_resp;
 341         dma_addr_t bch_rhandle;
 342
 343         /* BCH of all-0xff, so erased pages read as error-free */
 344         unsigned char *eccmask;
 345 };
 346
 347 /* settable timings - 0..7 select timing of alen1..4/clen1..3/etc */
 348 enum tm_idx {
 349         t0, /* fixed at 4<<mult cycles */
 350         t1, t2, t3, t4, t5, t6, t7, /* settable per ONFI-timing mode */
 351 };
 352
 353 struct octeontx_probe_device {
 354         struct list_head list;
 355         struct udevice *dev;
 356 };
 357
 358 static struct bch_vf *bch_vf;
 359 /** Deferred devices due to BCH not being ready */
 360 LIST_HEAD(octeontx_pci_nand_deferred_devices);
 361
 362 /** default parameters used for probing chips */
 363 #define MAX_ONFI_MODE   5
 364
 365 static int default_onfi_timing;
 366 static int slew_ns = 2; /* default timing padding */
 367 static int def_ecc_size = 512; /* 1024 best for sw_bch, <= 4095 for hw_bch */
 368 static int default_width = 1; /* 8 bit */
 369 static int default_page_size = 2048;
 370 static struct ndf_set_tm_par_cmd default_timing_parms;
 371
 372 /** Port from Linux */
 373 #define readq_poll_timeout(addr, val, cond, delay_us, timeout_us)       \
 374 ({                                                                      \
 375         ulong __start = get_timer(0);                                   \
 376         void *__addr = (addr);                                          \
 377         const ulong __timeout_ms = timeout_us / 1000;                   \
 378         do {                                                            \
 379                 (val) = readq(__addr);                                  \
 380                 if (cond)                                               \
 381                         break;                                          \
 382                 if (timeout_us && get_timer(__start) > __timeout_ms) {  \
 383                         (val) = readq(__addr);                          \
 384                         break;                                          \
 385                 }                                                       \
 386                 if (delay_us)                                           \
 387                         udelay(delay_us);                               \
 388         } while (1);                                                    \
 389         (cond) ? 0 : -ETIMEDOUT;                                        \
 390 })
 391
 392 /** Ported from Linux 4.9.0 include/linux/of.h for compatibility */
 393 static inline int of_get_child_count(const ofnode node)
 394 {
 395         return fdtdec_get_child_count(gd->fdt_blob, ofnode_to_offset(node));
 396 }
 397
 398 /**
 399  * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
 400  */
 401 static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
 402                                  struct mtd_oob_region *oobregion)
 403 {
 404         struct nand_chip *chip = mtd_to_nand(mtd);
 405         struct nand_ecc_ctrl *ecc = &chip->ecc;
 406
 407         if (section || !ecc->total)
 408                 return -ERANGE;
 409
 410         oobregion->length = ecc->total;
 411         oobregion->offset = mtd->oobsize - oobregion->length;
 412
 413         return 0;
 414 }
 415
 416 /**
 417  * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
 418  */
 419 static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
 420                                   struct mtd_oob_region *oobregion)
 421 {
 422         struct nand_chip *chip = mtd_to_nand(mtd);
 423         struct nand_ecc_ctrl *ecc = &chip->ecc;
 424
 425         if (section)
 426                 return -ERANGE;
 427
 428         oobregion->length = mtd->oobsize - ecc->total - 2;
 429         oobregion->offset = 2;
 430
 431         return 0;
 432 }
 433
 434 static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
 435         .ecc = nand_ooblayout_ecc_lp,
 436         .rfree = nand_ooblayout_free_lp,
 437 };
 438
 439 static inline struct octeontx_nand_chip *to_otx_nand(struct nand_chip *nand)
 440 {
 441         return container_of(nand, struct octeontx_nand_chip, nand);
 442 }
 443
 444 static inline struct octeontx_nfc *to_otx_nfc(struct nand_hw_control *ctrl)
 445 {
 446         return container_of(ctrl, struct octeontx_nfc, controller);
 447 }
 448
 449 static int octeontx_nand_calc_ecc_layout(struct nand_chip *nand)
 450 {
 451         struct nand_ecclayout *layout = nand->ecc.layout;
 452         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 453         struct mtd_info *mtd = &nand->mtd;
 454         int oobsize = mtd->oobsize;
 455         int i;
 456         bool layout_alloc = false;
 457
 458         if (!layout) {
 459                 layout = devm_kzalloc(tn->dev, sizeof(*layout), GFP_KERNEL);
 460                 if (!layout)
 461                         return -ENOMEM;
 462                 nand->ecc.layout = layout;
 463                 layout_alloc = true;
 464         }
 465         layout->eccbytes = nand->ecc.steps * nand->ecc.bytes;
 466         /* Reserve 2 bytes for bad block marker */
 467         if (layout->eccbytes + 2 > oobsize) {
 468                 pr_err("No suitable oob scheme available for oobsize %d eccbytes %u\n",
 469                        oobsize, layout->eccbytes);
 470                 goto fail;
 471         }
 472         /* put ecc bytes at oob tail */
 473         for (i = 0; i < layout->eccbytes; i++)
 474                 layout->eccpos[i] = oobsize - layout->eccbytes + i;
 475         layout->oobfree[0].offset = 2;
 476         layout->oobfree[0].length = oobsize - 2 - layout->eccbytes;
 477         nand->ecc.layout = layout;
 478         return 0;
 479
 480 fail:
 481         if (layout_alloc)
 482                 kfree(layout);
 483         return -1;
 484 }
 485
 486 /*
 487  * Read a single byte from the temporary buffer. Used after READID
 488  * to get the NAND information and for STATUS.
 489  */
 490 static u8 octeontx_nand_read_byte(struct mtd_info *mtd)
 491 {
 492         struct nand_chip *nand = mtd_to_nand(mtd);
 493         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 494
 495         if (tn->use_status) {
 496                 tn->use_status = false;
 497                 return *tn->stat;
 498         }
 499
 500         if (tn->buf.data_index < tn->buf.data_len)
 501                 return tn->buf.dmabuf[tn->buf.data_index++];
 502
 503         dev_err(tn->dev, "No data to read, idx: 0x%x, len: 0x%x\n",
 504                 tn->buf.data_index, tn->buf.data_len);
 505
 506         return 0xff;
 507 }
 508
 509 /*
 510  * Read a number of pending bytes from the temporary buffer. Used
 511  * to get page and OOB data.
 512  */
 513 static void octeontx_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len)
 514 {
 515         struct nand_chip *nand = mtd_to_nand(mtd);
 516         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 517
 518         if (len > tn->buf.data_len - tn->buf.data_index) {
 519                 dev_err(tn->dev, "Not enough data for read of %d bytes\n", len);
 520                 return;
 521         }
 522
 523         memcpy(buf, tn->buf.dmabuf + tn->buf.data_index, len);
 524         tn->buf.data_index += len;
 525 }
 526
 527 static void octeontx_nand_write_buf(struct mtd_info *mtd,
 528                                     const u8 *buf, int len)
 529 {
 530         struct nand_chip *nand = mtd_to_nand(mtd);
 531         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 532
 533         memcpy(tn->buf.dmabuf + tn->buf.data_len, buf, len);
 534         tn->buf.data_len += len;
 535 }
 536
 537 /* Overwrite default function to avoid sync abort on chip = -1. */
 538 static void octeontx_nand_select_chip(struct mtd_info *mtd, int chip)
 539 {
 540 }
 541
 542 static inline int timing_to_cycle(u32 psec, unsigned long clock)
 543 {
 544         unsigned int ns;
 545         int ticks;
 546
 547         ns = DIV_ROUND_UP(psec, 1000);
 548         ns += slew_ns;
 549
 550         /* no rounding needed since clock is multiple of 1MHz */
 551         clock /= 1000000;
 552         ns *= clock;
 553
 554         ticks = DIV_ROUND_UP(ns, 1000);
 555
 556         /* actual delay is (tm_parX+1)<<tim_mult */
 557         if (ticks)
 558                 ticks--;
 559
 560         return ticks;
 561 }
 562
 563 static void set_timings(struct octeontx_nand_chip *chip,
 564                         struct ndf_set_tm_par_cmd *tp,
 565                         const struct nand_sdr_timings *timings,
 566                         unsigned long sclk)
 567 {
 568         /* scaled coprocessor-cycle values */
 569         u32 s_wh, s_cls, s_clh, s_rp, s_wb, s_wc;
 570
 571         tp->tim_mult = 0;
 572         s_wh = timing_to_cycle(timings->tWH_min, sclk);
 573         s_cls = timing_to_cycle(timings->tCLS_min, sclk);
 574         s_clh = timing_to_cycle(timings->tCLH_min, sclk);
 575         s_rp = timing_to_cycle(timings->tRP_min, sclk);
 576         s_wb = timing_to_cycle(timings->tWB_max, sclk);
 577         s_wc = timing_to_cycle(timings->tWC_min, sclk);
 578
 579         tp->tm_par1 = s_wh;
 580         tp->tm_par2 = s_clh;
 581         tp->tm_par3 = s_rp + 1;
 582         tp->tm_par4 = s_cls - s_wh;
 583         tp->tm_par5 = s_wc - s_wh + 1;
 584         tp->tm_par6 = s_wb;
 585         tp->tm_par7 = 0;
 586         tp->tim_mult++; /* overcompensate for bad math */
 587
 588         /* TODO: comment parameter re-use */
 589
 590         pr_debug("%s: tim_par: mult: %d  p1: %d  p2: %d  p3: %d\n",
 591                  __func__, tp->tim_mult, tp->tm_par1, tp->tm_par2, tp->tm_par3);
 592         pr_debug("                 p4: %d  p5: %d  p6: %d  p7: %d\n",
 593                  tp->tm_par4, tp->tm_par5, tp->tm_par6, tp->tm_par7);
 594 }
 595
 596 static int set_default_timings(struct octeontx_nfc *tn,
 597                                const struct nand_sdr_timings *timings)
 598 {
 599         unsigned long sclk = octeontx_get_io_clock();
 600
 601         set_timings(NULL, &default_timing_parms, timings, sclk);
 602         return 0;
 603 }
 604
 605 static int octeontx_nfc_chip_set_timings(struct octeontx_nand_chip *chip,
 606                                          const struct nand_sdr_timings *timings)
 607 {
 608         /*struct octeontx_nfc *tn = to_otx_nfc(chip->nand.controller);*/
 609         unsigned long sclk = octeontx_get_io_clock();
 610
 611         set_timings(chip, &chip->timings, timings, sclk);
 612         return 0;
 613 }
 614
 615 /* How many bytes are free in the NFD_CMD queue? */
 616 static int ndf_cmd_queue_free(struct octeontx_nfc *tn)
 617 {
 618         u64 ndf_misc;
 619
 620         ndf_misc = readq(tn->base + NDF_MISC);
 621         return FIELD_GET(NDF_MISC_FR_BYTE, ndf_misc);
 622 }
 623
 624 /* Submit a command to the NAND command queue. */
 625 static int ndf_submit(struct octeontx_nfc *tn, union ndf_cmd *cmd)
 626 {
 627         int opcode = cmd->val[0] & 0xf;
 628
 629         switch (opcode) {
 630         /* All these commands fit in one 64bit word */
 631         case NDF_OP_NOP:
 632         case NDF_OP_SET_TM_PAR:
 633         case NDF_OP_WAIT:
 634         case NDF_OP_CHIP_EN_DIS:
 635         case NDF_OP_CLE_CMD:
 636         case NDF_OP_WR_CMD:
 637         case NDF_OP_RD_CMD:
 638         case NDF_OP_RD_EDO_CMD:
 639         case NDF_OP_BUS_ACQ_REL:
 640                 if (ndf_cmd_queue_free(tn) < 8)
 641                         goto full;
 642                 writeq(cmd->val[0], tn->base + NDF_CMD);
 643                 break;
 644         case NDF_OP_ALE_CMD:
 645                 /* ALE commands take either one or two 64bit words */
 646                 if (cmd->u.ale_cmd.adr_byte_num < 5) {
 647                         if (ndf_cmd_queue_free(tn) < 8)
 648                                 goto full;
 649                         writeq(cmd->val[0], tn->base + NDF_CMD);
 650                 } else {
 651                         if (ndf_cmd_queue_free(tn) < 16)
 652                                 goto full;
 653                         writeq(cmd->val[0], tn->base + NDF_CMD);
 654                         writeq(cmd->val[1], tn->base + NDF_CMD);
 655                 }
 656                 break;
 657         case NDF_OP_WAIT_STATUS: /* Wait status commands take two 64bit words */
 658                 if (ndf_cmd_queue_free(tn) < 16)
 659                         goto full;
 660                 writeq(cmd->val[0], tn->base + NDF_CMD);
 661                 writeq(cmd->val[1], tn->base + NDF_CMD);
 662                 break;
 663         default:
 664                 dev_err(tn->dev, "%s: unknown command: %u\n", __func__, opcode);
 665                 return -EINVAL;
 666         }
 667         return 0;
 668
 669 full:
 670         dev_err(tn->dev, "%s: no space left in command queue\n", __func__);
 671         return -ENOMEM;
 672 }
 673
 674 /**
 675  * Wait for the ready/busy signal. First wait for busy to be valid,
 676  * then wait for busy to de-assert.
 677  */
 678 static int ndf_build_wait_busy(struct octeontx_nfc *tn)
 679 {
 680         union ndf_cmd cmd;
 681
 682         memset(&cmd, 0, sizeof(cmd));
 683         cmd.u.wait.opcode = NDF_OP_WAIT;
 684         cmd.u.wait.r_b = 1;
 685         cmd.u.wait.wlen = t6;
 686
 687         if (ndf_submit(tn, &cmd))
 688                 return -ENOMEM;
 689         return 0;
 690 }
 691
 692 static bool ndf_dma_done(struct octeontx_nfc *tn)
 693 {
 694         u64 dma_cfg;
 695
 696         /* Enable bit should be clear after a transfer */
 697         dma_cfg = readq(tn->base + NDF_DMA_CFG);
 698         if (!(dma_cfg & NDF_DMA_CFG_EN))
 699                 return true;
 700
 701         return false;
 702 }
 703
 704 static int ndf_wait(struct octeontx_nfc *tn)
 705 {
 706         ulong start = get_timer(0);
 707         bool done;
 708
 709         while (!(done = ndf_dma_done(tn)) && get_timer(start) < NDF_TIMEOUT)
 710                 ;
 711
 712         if (!done) {
 713                 dev_err(tn->dev, "%s: timeout error\n", __func__);
 714                 return -ETIMEDOUT;
 715         }
 716         return 0;
 717 }
 718
 719 static int ndf_wait_idle(struct octeontx_nfc *tn)
 720 {
 721         u64 val;
 722         u64 dval = 0;
 723         int rc;
 724         int pause = 100;
 725         u64 tot_us = USEC_PER_SEC / 10;
 726
 727         rc = readq_poll_timeout(tn->base + NDF_ST_REG,
 728                                 val, val & NDF_ST_REG_EXE_IDLE, pause, tot_us);
 729         if (!rc)
 730                 rc = readq_poll_timeout(tn->base + NDF_DMA_CFG,
 731                                         dval, !(dval & NDF_DMA_CFG_EN),
 732                                         pause, tot_us);
 733
 734         return rc;
 735 }
 736
 737 /** Issue set timing parameters */
 738 static int ndf_queue_cmd_timing(struct octeontx_nfc *tn,
 739                                 struct ndf_set_tm_par_cmd *timings)
 740 {
 741         union ndf_cmd cmd;
 742
 743         memset(&cmd, 0, sizeof(cmd));
 744         cmd.u.set_tm_par.opcode = NDF_OP_SET_TM_PAR;
 745         cmd.u.set_tm_par.tim_mult = timings->tim_mult;
 746         cmd.u.set_tm_par.tm_par1 = timings->tm_par1;
 747         cmd.u.set_tm_par.tm_par2 = timings->tm_par2;
 748         cmd.u.set_tm_par.tm_par3 = timings->tm_par3;
 749         cmd.u.set_tm_par.tm_par4 = timings->tm_par4;
 750         cmd.u.set_tm_par.tm_par5 = timings->tm_par5;
 751         cmd.u.set_tm_par.tm_par6 = timings->tm_par6;
 752         cmd.u.set_tm_par.tm_par7 = timings->tm_par7;
 753         return ndf_submit(tn, &cmd);
 754 }
 755
 756 /** Issue bus acquire or release */
 757 static int ndf_queue_cmd_bus(struct octeontx_nfc *tn, int direction)
 758 {
 759         union ndf_cmd cmd;
 760
 761         memset(&cmd, 0, sizeof(cmd));
 762         cmd.u.bus_acq_rel.opcode = NDF_OP_BUS_ACQ_REL;
 763         cmd.u.bus_acq_rel.direction = direction;
 764         return ndf_submit(tn, &cmd);
 765 }
 766
 767 /* Issue chip select or deselect */
 768 static int ndf_queue_cmd_chip(struct octeontx_nfc *tn, int enable, int chip,
 769                               int width)
 770 {
 771         union ndf_cmd cmd;
 772
 773         memset(&cmd, 0, sizeof(cmd));
 774         cmd.u.chip_en_dis.opcode = NDF_OP_CHIP_EN_DIS;
 775         cmd.u.chip_en_dis.chip = chip;
 776         cmd.u.chip_en_dis.enable = enable;
 777         cmd.u.chip_en_dis.bus_width = width;
 778         return ndf_submit(tn, &cmd);
 779 }
 780
 781 static int ndf_queue_cmd_wait(struct octeontx_nfc *tn, int t_delay)
 782 {
 783         union ndf_cmd cmd;
 784
 785         memset(&cmd, 0, sizeof(cmd));
 786         cmd.u.wait.opcode = NDF_OP_WAIT;
 787         cmd.u.wait.wlen = t_delay;
 788         return ndf_submit(tn, &cmd);
 789 }
 790
 791 static int ndf_queue_cmd_cle(struct octeontx_nfc *tn, int command)
 792 {
 793         union ndf_cmd cmd;
 794
 795         memset(&cmd, 0, sizeof(cmd));
 796         cmd.u.cle_cmd.opcode = NDF_OP_CLE_CMD;
 797         cmd.u.cle_cmd.cmd_data = command;
 798         cmd.u.cle_cmd.clen1 = t4;
 799         cmd.u.cle_cmd.clen2 = t1;
 800         cmd.u.cle_cmd.clen3 = t2;
 801         return ndf_submit(tn, &cmd);
 802 }
 803
 804 static int ndf_queue_cmd_ale(struct octeontx_nfc *tn, int addr_bytes,
 805                              struct nand_chip *nand, u64 page,
 806                              u32 col, int page_size)
 807 {
 808         struct octeontx_nand_chip *octeontx_nand = (nand) ?
 809                                                 to_otx_nand(nand) : NULL;
 810         union ndf_cmd cmd;
 811
 812         memset(&cmd, 0, sizeof(cmd));
 813         cmd.u.ale_cmd.opcode = NDF_OP_ALE_CMD;
 814         cmd.u.ale_cmd.adr_byte_num = addr_bytes;
 815
 816         /* set column bit for OOB area, assume OOB follows page */
 817         if (octeontx_nand && octeontx_nand->oob_only)
 818                 col += page_size;
 819
 820         /* page is u64 for this generality, even if cmdfunc() passes int */
 821         switch (addr_bytes) {
 822         /* 4-8 bytes: page, then 2-byte col */
 823         case 8:
 824                 cmd.u.ale_cmd.adr_byt8 = (page >> 40) & 0xff;
 825                 fallthrough;
 826         case 7:
 827                 cmd.u.ale_cmd.adr_byt7 = (page >> 32) & 0xff;
 828                 fallthrough;
 829         case 6:
 830                 cmd.u.ale_cmd.adr_byt6 = (page >> 24) & 0xff;
 831                 fallthrough;
 832         case 5:
 833                 cmd.u.ale_cmd.adr_byt5 = (page >> 16) & 0xff;
 834                 fallthrough;
 835         case 4:
 836                 cmd.u.ale_cmd.adr_byt4 = (page >> 8) & 0xff;
 837                 cmd.u.ale_cmd.adr_byt3 = page & 0xff;
 838                 cmd.u.ale_cmd.adr_byt2 = (col >> 8) & 0xff;
 839                 cmd.u.ale_cmd.adr_byt1 =  col & 0xff;
 840                 break;
 841         /* 1-3 bytes: just the page address */
 842         case 3:
 843                 cmd.u.ale_cmd.adr_byt3 = (page >> 16) & 0xff;
 844                 fallthrough;
 845         case 2:
 846                 cmd.u.ale_cmd.adr_byt2 = (page >> 8) & 0xff;
 847                 fallthrough;
 848         case 1:
 849                 cmd.u.ale_cmd.adr_byt1 = page & 0xff;
 850                 break;
 851         default:
 852                 break;
 853         }
 854
 855         cmd.u.ale_cmd.alen1 = t3;
 856         cmd.u.ale_cmd.alen2 = t1;
 857         cmd.u.ale_cmd.alen3 = t5;
 858         cmd.u.ale_cmd.alen4 = t2;
 859         return ndf_submit(tn, &cmd);
 860 }
 861
 862 static int ndf_queue_cmd_write(struct octeontx_nfc *tn, int len)
 863 {
 864         union ndf_cmd cmd;
 865
 866         memset(&cmd, 0, sizeof(cmd));
 867         cmd.u.wr_cmd.opcode = NDF_OP_WR_CMD;
 868         cmd.u.wr_cmd.data = len;
 869         cmd.u.wr_cmd.wlen1 = t3;
 870         cmd.u.wr_cmd.wlen2 = t1;
 871         return ndf_submit(tn, &cmd);
 872 }
 873
 874 static int ndf_build_pre_cmd(struct octeontx_nfc *tn, int cmd1,
 875                              int addr_bytes, u64 page, u32 col, int cmd2)
 876 {
 877         struct nand_chip *nand = tn->controller.active;
 878         struct octeontx_nand_chip *octeontx_nand;
 879         struct ndf_set_tm_par_cmd *timings;
 880         int width, page_size, rc;
 881
 882         /* Also called before chip probing is finished */
 883         if (!nand) {
 884                 timings = &default_timing_parms;
 885                 page_size = default_page_size;
 886                 width = default_width;
 887         } else {
 888                 octeontx_nand = to_otx_nand(nand);
 889                 timings = &octeontx_nand->timings;
 890                 page_size = nand->mtd.writesize;
 891                 if (nand->options & NAND_BUSWIDTH_16)
 892                         width = 2;
 893                 else
 894                         width = 1;
 895         }
 896         rc = ndf_queue_cmd_timing(tn, timings);
 897         if (rc)
 898                 return rc;
 899
 900         rc = ndf_queue_cmd_bus(tn, NDF_BUS_ACQUIRE);
 901         if (rc)
 902                 return rc;
 903
 904         rc = ndf_queue_cmd_chip(tn, 1, tn->selected_chip, width);
 905         if (rc)
 906                 return rc;
 907
 908         rc = ndf_queue_cmd_wait(tn, t1);
 909         if (rc)
 910                 return rc;
 911
 912         rc = ndf_queue_cmd_cle(tn, cmd1);
 913         if (rc)
 914                 return rc;
 915
 916         if (addr_bytes) {
 917                 rc = ndf_build_wait_busy(tn);
 918                 if (rc)
 919                         return rc;
 920
 921                 rc = ndf_queue_cmd_ale(tn, addr_bytes, nand,
 922                                        page, col, page_size);
 923                 if (rc)
 924                         return rc;
 925         }
 926
 927         /* CLE 2 */
 928         if (cmd2) {
 929                 rc = ndf_build_wait_busy(tn);
 930                 if (rc)
 931                         return rc;
 932
 933                 rc = ndf_queue_cmd_cle(tn, cmd2);
 934                 if (rc)
 935                         return rc;
 936         }
 937         return 0;
 938 }
 939
 940 static int ndf_build_post_cmd(struct octeontx_nfc *tn, int hold_time)
 941 {
 942         int rc;
 943
 944         /* Deselect chip */
 945         rc = ndf_queue_cmd_chip(tn, 0, 0, 0);
 946         if (rc)
 947                 return rc;
 948
 949         rc = ndf_queue_cmd_wait(tn, t2);
 950         if (rc)
 951                 return rc;
 952
 953         /* Release bus */
 954         rc = ndf_queue_cmd_bus(tn, 0);
 955         if (rc)
 956                 return rc;
 957
 958         rc = ndf_queue_cmd_wait(tn, hold_time);
 959         if (rc)
 960                 return rc;
 961
 962         /*
 963          * Last action is ringing the doorbell with number of bus
 964          * acquire-releases cycles (currently 1).
 965          */
 966         writeq(1, tn->base + NDF_DRBELL);
 967         return 0;
 968 }
 969
 970 /* Setup the NAND DMA engine for a transfer. */
 971 static void ndf_setup_dma(struct octeontx_nfc *tn, int is_write,
 972                           dma_addr_t bus_addr, int len)
 973 {
 974         u64 dma_cfg;
 975
 976         dma_cfg = FIELD_PREP(NDF_DMA_CFG_RW, is_write) |
 977                   FIELD_PREP(NDF_DMA_CFG_SIZE, (len >> 3) - 1);
 978         dma_cfg |= NDF_DMA_CFG_EN;
 979         writeq(bus_addr, tn->base + NDF_DMA_ADR);
 980         writeq(dma_cfg, tn->base + NDF_DMA_CFG);
 981 }
 982
 983 static int octeontx_nand_reset(struct octeontx_nfc *tn)
 984 {
 985         int rc;
 986
 987         rc = ndf_build_pre_cmd(tn, NAND_CMD_RESET, 0, 0, 0, 0);
 988         if (rc)
 989                 return rc;
 990
 991         rc = ndf_build_wait_busy(tn);
 992         if (rc)
 993                 return rc;
 994
 995         rc = ndf_build_post_cmd(tn, t2);
 996         if (rc)
 997                 return rc;
 998
 999         return 0;
1000 }
1001
1002 static int ndf_read(struct octeontx_nfc *tn, int cmd1, int addr_bytes,
1003                     u64 page, u32 col, int cmd2, int len)
1004 {
1005         dma_addr_t bus_addr = tn->use_status ? tn->stat_addr : tn->buf.dmaaddr;
1006         struct nand_chip *nand = tn->controller.active;
1007         int timing_mode, bytes, rc;
1008         union ndf_cmd cmd;
1009         u64 start, end;
1010
1011         pr_debug("%s(%p, 0x%x, 0x%x, 0x%llx, 0x%x, 0x%x, 0x%x)\n", __func__,
1012                  tn, cmd1, addr_bytes, page, col, cmd2, len);
1013         if (!nand)
1014                 timing_mode = default_onfi_timing;
1015         else
1016                 timing_mode = nand->onfi_timing_mode_default;
1017
1018         /* Build the command and address cycles */
1019         rc = ndf_build_pre_cmd(tn, cmd1, addr_bytes, page, col, cmd2);
1020         if (rc) {
1021                 dev_err(tn->dev, "Build pre command failed\n");
1022                 return rc;
1023         }
1024
1025         /* This waits for some time, then waits for busy to be de-asserted. */
1026         rc = ndf_build_wait_busy(tn);
1027         if (rc) {
1028                 dev_err(tn->dev, "Wait timeout\n");
1029                 return rc;
1030         }
1031
1032         memset(&cmd, 0, sizeof(cmd));
1033
1034         if (timing_mode < 4)
1035                 cmd.u.rd_cmd.opcode = NDF_OP_RD_CMD;
1036         else
1037                 cmd.u.rd_cmd.opcode = NDF_OP_RD_EDO_CMD;
1038
1039         cmd.u.rd_cmd.data = len;
1040         cmd.u.rd_cmd.rlen1 = t7;
1041         cmd.u.rd_cmd.rlen2 = t3;
1042         cmd.u.rd_cmd.rlen3 = t1;
1043         cmd.u.rd_cmd.rlen4 = t7;
1044         rc = ndf_submit(tn, &cmd);
1045         if (rc) {
1046                 dev_err(tn->dev, "Error submitting command\n");
1047                 return rc;
1048         }
1049
1050         start = (u64)bus_addr;
1051         ndf_setup_dma(tn, 0, bus_addr, len);
1052
1053         rc = ndf_build_post_cmd(tn, t2);
1054         if (rc) {
1055                 dev_err(tn->dev, "Build post command failed\n");
1056                 return rc;
1057         }
1058
1059         /* Wait for the DMA to complete */
1060         rc = ndf_wait(tn);
1061         if (rc) {
1062                 dev_err(tn->dev, "DMA timed out\n");
1063                 return rc;
1064         }
1065
1066         end = readq(tn->base + NDF_DMA_ADR);
1067         bytes = end - start;
1068
1069         /* Make sure NDF is really done */
1070         rc = ndf_wait_idle(tn);
1071         if (rc) {
1072                 dev_err(tn->dev, "poll idle failed\n");
1073                 return rc;
1074         }
1075
1076         pr_debug("%s: Read %d bytes\n", __func__, bytes);
1077         return bytes;
1078 }
1079
1080 static int octeontx_nand_get_features(struct mtd_info *mtd,
1081                                       struct nand_chip *chip, int feature_addr,
1082                                       u8 *subfeature_para)
1083 {
1084         struct nand_chip *nand = chip;
1085         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1086         int len = 8;
1087         int rc;
1088
1089         pr_debug("%s: feature addr: 0x%x\n", __func__, feature_addr);
1090         memset(tn->buf.dmabuf, 0xff, len);
1091         tn->buf.data_index = 0;
1092         tn->buf.data_len = 0;
1093         rc = ndf_read(tn, NAND_CMD_GET_FEATURES, 1, feature_addr, 0, 0, len);
1094         if (rc)
1095                 return rc;
1096
1097         memcpy(subfeature_para, tn->buf.dmabuf, ONFI_SUBFEATURE_PARAM_LEN);
1098
1099         return 0;
1100 }
1101
1102 static int octeontx_nand_set_features(struct mtd_info *mtd,
1103                                       struct nand_chip *chip, int feature_addr,
1104                                       u8 *subfeature_para)
1105 {
1106         struct nand_chip *nand = chip;
1107         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1108         const int len = ONFI_SUBFEATURE_PARAM_LEN;
1109         int rc;
1110
1111         rc = ndf_build_pre_cmd(tn, NAND_CMD_SET_FEATURES,
1112                                1, feature_addr, 0, 0);
1113         if (rc)
1114                 return rc;
1115
1116         memcpy(tn->buf.dmabuf, subfeature_para, len);
1117         memset(tn->buf.dmabuf + len, 0, 8 - len);
1118
1119         ndf_setup_dma(tn, 1, tn->buf.dmaaddr, 8);
1120
1121         rc = ndf_queue_cmd_write(tn, 8);
1122         if (rc)
1123                 return rc;
1124
1125         rc = ndf_build_wait_busy(tn);
1126         if (rc)
1127                 return rc;
1128
1129         rc = ndf_build_post_cmd(tn, t2);
1130         if (rc)
1131                 return rc;
1132
1133         return 0;
1134 }
1135
1136 /*
1137  * Read a page from NAND. If the buffer has room, the out of band
1138  * data will be included.
1139  */
1140 static int ndf_page_read(struct octeontx_nfc *tn, u64 page, int col, int len)
1141 {
1142         debug("%s(%p, 0x%llx, 0x%x, 0x%x) active: %p\n", __func__,
1143               tn, page, col, len, tn->controller.active);
1144         struct nand_chip *nand = tn->controller.active;
1145         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1146         int addr_bytes = chip->row_bytes + chip->col_bytes;
1147
1148         memset(tn->buf.dmabuf, 0xff, len);
1149         return ndf_read(tn, NAND_CMD_READ0, addr_bytes,
1150                     page, col, NAND_CMD_READSTART, len);
1151 }
1152
1153 /* Erase a NAND block */
1154 static int ndf_block_erase(struct octeontx_nfc *tn, u64 page_addr)
1155 {
1156         struct nand_chip *nand = tn->controller.active;
1157         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1158         int addr_bytes = chip->row_bytes;
1159         int rc;
1160
1161         rc = ndf_build_pre_cmd(tn, NAND_CMD_ERASE1, addr_bytes,
1162                                page_addr, 0, NAND_CMD_ERASE2);
1163         if (rc)
1164                 return rc;
1165
1166         /* Wait for R_B to signal erase is complete  */
1167         rc = ndf_build_wait_busy(tn);
1168         if (rc)
1169                 return rc;
1170
1171         rc = ndf_build_post_cmd(tn, t2);
1172         if (rc)
1173                 return rc;
1174
1175         /* Wait until the command queue is idle */
1176         return ndf_wait_idle(tn);
1177 }
1178
1179 /*
1180  * Write a page (or less) to NAND.
1181  */
1182 static int ndf_page_write(struct octeontx_nfc *tn, int page)
1183 {
1184         int len, rc;
1185         struct nand_chip *nand = tn->controller.active;
1186         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1187         int addr_bytes = chip->row_bytes + chip->col_bytes;
1188
1189         len = tn->buf.data_len - tn->buf.data_index;
1190         chip->oob_only = (tn->buf.data_index >= nand->mtd.writesize);
1191         WARN_ON_ONCE(len & 0x7);
1192
1193         ndf_setup_dma(tn, 1, tn->buf.dmaaddr + tn->buf.data_index, len);
1194         rc = ndf_build_pre_cmd(tn, NAND_CMD_SEQIN, addr_bytes, page, 0, 0);
1195         if (rc)
1196                 return rc;
1197
1198         rc = ndf_queue_cmd_write(tn, len);
1199         if (rc)
1200                 return rc;
1201
1202         rc = ndf_queue_cmd_cle(tn, NAND_CMD_PAGEPROG);
1203         if (rc)
1204                 return rc;
1205
1206         /* Wait for R_B to signal program is complete  */
1207         rc = ndf_build_wait_busy(tn);
1208         if (rc)
1209                 return rc;
1210
1211         rc = ndf_build_post_cmd(tn, t2);
1212         if (rc)
1213                 return rc;
1214
1215         /* Wait for the DMA to complete */
1216         rc = ndf_wait(tn);
1217         if (rc)
1218                 return rc;
1219
1220         /* Data transfer is done but NDF is not, it is waiting for R/B# */
1221         return ndf_wait_idle(tn);
1222 }
1223
1224 static void octeontx_nand_cmdfunc(struct mtd_info *mtd, unsigned int command,
1225                                   int column, int page_addr)
1226 {
1227         struct nand_chip *nand = mtd_to_nand(mtd);
1228         struct octeontx_nand_chip *octeontx_nand = to_otx_nand(nand);
1229         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1230         int rc;
1231
1232         tn->selected_chip = octeontx_nand->cs;
1233         if (tn->selected_chip < 0 || tn->selected_chip >= NAND_MAX_CHIPS) {
1234                 dev_err(tn->dev, "invalid chip select\n");
1235                 return;
1236         }
1237
1238         tn->use_status = false;
1239
1240         pr_debug("%s(%p, 0x%x, 0x%x, 0x%x) cs: %d\n", __func__, mtd, command,
1241                  column, page_addr, tn->selected_chip);
1242         switch (command) {
1243         case NAND_CMD_READID:
1244                 tn->buf.data_index = 0;
1245                 octeontx_nand->oob_only = false;
1246                 rc = ndf_read(tn, command, 1, column, 0, 0, 8);
1247                 if (rc < 0)
1248                         dev_err(tn->dev, "READID failed with %d\n", rc);
1249                 else
1250                         tn->buf.data_len = rc;
1251                 break;
1252
1253         case NAND_CMD_READOOB:
1254                 octeontx_nand->oob_only = true;
1255                 tn->buf.data_index = 0;
1256                 tn->buf.data_len = 0;
1257                 rc = ndf_page_read(tn, page_addr, column, mtd->oobsize);
1258                 if (rc < mtd->oobsize)
1259                         dev_err(tn->dev, "READOOB failed with %d\n",
1260                                 tn->buf.data_len);
1261                 else
1262                         tn->buf.data_len = rc;
1263                 break;
1264
1265         case NAND_CMD_READ0:
1266                 octeontx_nand->oob_only = false;
1267                 tn->buf.data_index = 0;
1268                 tn->buf.data_len = 0;
1269                 rc = ndf_page_read(tn, page_addr, column,
1270                                    mtd->writesize + mtd->oobsize);
1271
1272                 if (rc < mtd->writesize + mtd->oobsize)
1273                         dev_err(tn->dev, "READ0 failed with %d\n", rc);
1274                 else
1275                         tn->buf.data_len = rc;
1276                 break;
1277
1278         case NAND_CMD_STATUS:
1279                 /* used in oob/not states */
1280                 tn->use_status = true;
1281                 rc = ndf_read(tn, command, 0, 0, 0, 0, 8);
1282                 if (rc < 0)
1283                         dev_err(tn->dev, "STATUS failed with %d\n", rc);
1284                 break;
1285
1286         case NAND_CMD_RESET:
1287                 /* used in oob/not states */
1288                 rc = octeontx_nand_reset(tn);
1289                 if (rc < 0)
1290                         dev_err(tn->dev, "RESET failed with %d\n", rc);
1291                 break;
1292
1293         case NAND_CMD_PARAM:
1294                 octeontx_nand->oob_only = false;
1295                 tn->buf.data_index = 0;
1296                 rc = ndf_read(tn, command, 1, 0, 0, 0,
1297                               min(tn->buf.dmabuflen, 3 * 512));
1298                 if (rc < 0)
1299                         dev_err(tn->dev, "PARAM failed with %d\n", rc);
1300                 else
1301                         tn->buf.data_len = rc;
1302                 break;
1303
1304         case NAND_CMD_RNDOUT:
1305                 tn->buf.data_index = column;
1306                 break;
1307
1308         case NAND_CMD_ERASE1:
1309                 if (ndf_block_erase(tn, page_addr))
1310                         dev_err(tn->dev, "ERASE1 failed\n");
1311                 break;
1312
1313         case NAND_CMD_ERASE2:
1314                 /* We do all erase processing in the first command, so ignore
1315                  * this one.
1316                  */
1317                 break;
1318
1319         case NAND_CMD_SEQIN:
1320                 octeontx_nand->oob_only = (column >= mtd->writesize);
1321                 tn->buf.data_index = column;
1322                 tn->buf.data_len = column;
1323
1324                 octeontx_nand->selected_page = page_addr;
1325                 break;
1326
1327         case NAND_CMD_PAGEPROG:
1328                 rc = ndf_page_write(tn, octeontx_nand->selected_page);
1329                 if (rc)
1330                         dev_err(tn->dev, "PAGEPROG failed with %d\n", rc);
1331                 break;
1332
1333         case NAND_CMD_SET_FEATURES:
1334                 octeontx_nand->oob_only = false;
1335                 /* assume tn->buf.data_len == 4 of data has been set there */
1336                 rc = octeontx_nand_set_features(mtd, nand,
1337                                                 page_addr, tn->buf.dmabuf);
1338                 if (rc)
1339                         dev_err(tn->dev, "SET_FEATURES failed with %d\n", rc);
1340                 break;
1341
1342         case NAND_CMD_GET_FEATURES:
1343                 octeontx_nand->oob_only = false;
1344                 rc = octeontx_nand_get_features(mtd, nand,
1345                                                 page_addr, tn->buf.dmabuf);
1346                 if (!rc) {
1347                         tn->buf.data_index = 0;
1348                         tn->buf.data_len = 4;
1349                 } else {
1350                         dev_err(tn->dev, "GET_FEATURES failed with %d\n", rc);
1351                 }
1352                 break;
1353
1354         default:
1355                 WARN_ON_ONCE(1);
1356                 dev_err(tn->dev, "unhandled nand cmd: %x\n", command);
1357         }
1358 }
1359
1360 static int octeontx_nand_waitfunc(struct mtd_info *mtd, struct nand_chip *chip)
1361 {
1362         struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
1363         int ret;
1364
1365         ret = ndf_wait_idle(tn);
1366         return (ret < 0) ? -EIO : 0;
1367 }
1368
1369 /* check compatibility with ONFI timing mode#N, and optionally apply */
1370 /* TODO: Implement chipnr support? */
1371 static int octeontx_nand_setup_dat_intf(struct mtd_info *mtd, int chipnr,
1372                                         const struct nand_data_interface *conf)
1373 {
1374         static const bool check_only;
1375         struct nand_chip *nand = mtd_to_nand(mtd);
1376         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1377         static u64 t_wc_n[MAX_ONFI_MODE + 2]; /* cache a mode signature */
1378         int mode; /* deduced mode number, for reporting and restricting */
1379         int rc;
1380
1381         /*
1382          * Cache timing modes for reporting, and reducing needless change.
1383          *
1384          * Challenge: caller does not pass ONFI mode#, but reporting the mode
1385          * and restricting to a maximum, or a list, are useful for diagnosing
1386          * new hardware.  So use tWC_min, distinct and monotonic across modes,
1387          * to discover the requested/accepted mode number
1388          */
1389         for (mode = MAX_ONFI_MODE; mode >= 0 && !t_wc_n[0]; mode--) {
1390                 const struct nand_sdr_timings *t;
1391
1392                 t = onfi_async_timing_mode_to_sdr_timings(mode);
1393                 if (!t)
1394                         continue;
1395                 t_wc_n[mode] = t->tWC_min;
1396         }
1397
1398         if (!conf) {
1399                 rc = -EINVAL;
1400         } else if (check_only) {
1401                 rc = 0;
1402         } else if (nand->data_interface &&
1403                         chip->iface_set && chip->iface_mode == mode) {
1404                 /*
1405                  * Cases:
1406                  * - called from nand_reset, which clears DDR timing
1407                  *   mode back to SDR.  BUT if we're already in SDR,
1408                  *   timing mode persists over resets.
1409                  *   While mtd/nand layer only supports SDR,
1410                  *   this is always safe. And this driver only supports SDR.
1411                  *
1412                  * - called from post-power-event nand_reset (maybe
1413                  *   NFC+flash power down, or system hibernate.
1414                  *   Address this when CONFIG_PM support added
1415                  */
1416                 rc = 0;
1417         } else {
1418                 rc = octeontx_nfc_chip_set_timings(chip, &conf->timings.sdr);
1419                 if (!rc) {
1420                         chip->iface_mode = mode;
1421                         chip->iface_set = true;
1422                 }
1423         }
1424         return rc;
1425 }
1426
1427 static void octeontx_bch_reset(void)
1428 {
1429 }
1430
1431 /*
1432  * Given a page, calculate the ECC code
1433  *
1434  * chip:        Pointer to NAND chip data structure
1435  * buf:         Buffer to calculate ECC on
1436  * code:        Buffer to hold ECC data
1437  *
1438  * Return 0 on success or -1 on failure
1439  */
1440 static int octeontx_nand_bch_calculate_ecc_internal(struct mtd_info *mtd,
1441                                                     dma_addr_t ihandle,
1442                                                     u8 *code)
1443 {
1444         struct nand_chip *nand = mtd_to_nand(mtd);
1445         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1446         int rc;
1447         int i;
1448         static u8 *ecc_buffer;
1449         static int ecc_size;
1450         static unsigned long ecc_handle;
1451         union bch_resp *r = tn->bch_resp;
1452
1453         if (!ecc_buffer || ecc_size < nand->ecc.size) {
1454                 ecc_size = nand->ecc.size;
1455                 ecc_buffer = dma_alloc_coherent(ecc_size,
1456                                                 (unsigned long *)&ecc_handle);
1457         }
1458
1459         memset(ecc_buffer, 0, nand->ecc.bytes);
1460
1461         r->u16 = 0;
1462         __iowmb(); /* flush done=0 before making request */
1463
1464         rc = octeontx_bch_encode(bch_vf, ihandle, nand->ecc.size,
1465                                  nand->ecc.strength,
1466                                  (dma_addr_t)ecc_handle, tn->bch_rhandle);
1467
1468         if (!rc) {
1469                 octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
1470         } else {
1471                 dev_err(tn->dev, "octeontx_bch_encode failed\n");
1472                 return -1;
1473         }
1474
1475         if (!r->s.done || r->s.uncorrectable) {
1476                 dev_err(tn->dev,
1477                         "%s timeout, done:%d uncorr:%d corr:%d erased:%d\n",
1478                         __func__, r->s.done, r->s.uncorrectable,
1479                         r->s.num_errors, r->s.erased);
1480                 octeontx_bch_reset();
1481                 return -1;
1482         }
1483
1484         memcpy(code, ecc_buffer, nand->ecc.bytes);
1485
1486         for (i = 0; i < nand->ecc.bytes; i++)
1487                 code[i] ^= tn->eccmask[i];
1488
1489         return tn->bch_resp->s.num_errors;
1490 }
1491
1492 /*
1493  * Given a page, calculate the ECC code
1494  *
1495  * mtd:        MTD block structure
1496  * dat:        raw data (unused)
1497  * ecc_code:   buffer for ECC
1498  */
1499 static int octeontx_nand_bch_calculate(struct mtd_info *mtd,
1500                                        const u8 *dat, u8 *ecc_code)
1501 {
1502         struct nand_chip *nand = mtd_to_nand(mtd);
1503         dma_addr_t handle = dma_map_single((u8 *)dat,
1504                                            nand->ecc.size, DMA_TO_DEVICE);
1505         int ret;
1506
1507         ret = octeontx_nand_bch_calculate_ecc_internal(mtd, handle,
1508                                                        (void *)ecc_code);
1509
1510         return ret;
1511 }
1512
1513 /*
1514  * Detect and correct multi-bit ECC for a page
1515  *
1516  * mtd:        MTD block structure
1517  * dat:        raw data read from the chip
1518  * read_ecc:   ECC from the chip (unused)
1519  * isnull:     unused
1520  *
1521  * Returns number of bits corrected or -1 if unrecoverable
1522  */
1523 static int octeontx_nand_bch_correct(struct mtd_info *mtd, u_char *dat,
1524                                      u_char *read_ecc, u_char *isnull)
1525 {
1526         struct nand_chip *nand = mtd_to_nand(mtd);
1527         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1528         int i = nand->ecc.size + nand->ecc.bytes;
1529         static u8 *data_buffer;
1530         static dma_addr_t ihandle;
1531         static int buffer_size;
1532         dma_addr_t ohandle;
1533         union bch_resp *r = tn->bch_resp;
1534         int rc;
1535
1536         if (i > buffer_size) {
1537                 if (buffer_size)
1538                         free(data_buffer);
1539                 data_buffer = dma_alloc_coherent(i,
1540                                                  (unsigned long *)&ihandle);
1541                 if (!data_buffer) {
1542                         dev_err(tn->dev,
1543                                 "%s: Could not allocate %d bytes for buffer\n",
1544                                 __func__, i);
1545                         goto error;
1546                 }
1547                 buffer_size = i;
1548         }
1549
1550         memcpy(data_buffer, dat, nand->ecc.size);
1551         memcpy(data_buffer + nand->ecc.size, read_ecc, nand->ecc.bytes);
1552
1553         for (i = 0; i < nand->ecc.bytes; i++)
1554                 data_buffer[nand->ecc.size + i] ^= tn->eccmask[i];
1555
1556         r->u16 = 0;
1557         __iowmb(); /* flush done=0 before making request */
1558
1559         ohandle = dma_map_single(dat, nand->ecc.size, DMA_FROM_DEVICE);
1560         rc = octeontx_bch_decode(bch_vf, ihandle, nand->ecc.size,
1561                                  nand->ecc.strength, ohandle, tn->bch_rhandle);
1562
1563         if (!rc)
1564                 octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
1565
1566         if (rc) {
1567                 dev_err(tn->dev, "octeontx_bch_decode failed\n");
1568                 goto error;
1569         }
1570
1571         if (!r->s.done) {
1572                 dev_err(tn->dev, "Error: BCH engine timeout\n");
1573                 octeontx_bch_reset();
1574                 goto error;
1575         }
1576
1577         if (r->s.erased) {
1578                 debug("Info: BCH block is erased\n");
1579                 return 0;
1580         }
1581
1582         if (r->s.uncorrectable) {
1583                 debug("Cannot correct NAND block, response: 0x%x\n",
1584                       r->u16);
1585                 goto error;
1586         }
1587
1588         return r->s.num_errors;
1589
1590 error:
1591         debug("Error performing bch correction\n");
1592         return -1;
1593 }
1594
1595 void octeontx_nand_bch_hwctl(struct mtd_info *mtd, int mode)
1596 {
1597         /* Do nothing. */
1598 }
1599
1600 static int octeontx_nand_hw_bch_read_page(struct mtd_info *mtd,
1601                                           struct nand_chip *chip, u8 *buf,
1602                                           int oob_required, int page)
1603 {
1604         struct nand_chip *nand = mtd_to_nand(mtd);
1605         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1606         int i, eccsize = chip->ecc.size, ret;
1607         int eccbytes = chip->ecc.bytes;
1608         int eccsteps = chip->ecc.steps;
1609         u8 *p;
1610         u8 *ecc_code = chip->buffers->ecccode;
1611         unsigned int max_bitflips = 0;
1612
1613         /* chip->read_buf() insists on sequential order, we do OOB first */
1614         memcpy(chip->oob_poi, tn->buf.dmabuf + mtd->writesize, mtd->oobsize);
1615
1616         /* Use private buffer as input for ECC correction */
1617         p = tn->buf.dmabuf;
1618
1619         ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
1620                                          chip->ecc.total);
1621         if (ret)
1622                 return ret;
1623
1624         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1625                 int stat;
1626
1627                 debug("Correcting block offset %lx, ecc offset %x\n",
1628                       p - buf, i);
1629                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL);
1630
1631                 if (stat < 0) {
1632                         mtd->ecc_stats.failed++;
1633                         debug("Cannot correct NAND page %d\n", page);
1634                 } else {
1635                         mtd->ecc_stats.corrected += stat;
1636                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1637                 }
1638         }
1639
1640         /* Copy corrected data to caller's buffer now */
1641         memcpy(buf, tn->buf.dmabuf, mtd->writesize);
1642
1643         return max_bitflips;
1644 }
1645
1646 static int octeontx_nand_hw_bch_write_page(struct mtd_info *mtd,
1647                                            struct nand_chip *chip,
1648                                            const u8 *buf, int oob_required,
1649                                            int page)
1650 {
1651         struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
1652         int i, eccsize = chip->ecc.size, ret;
1653         int eccbytes = chip->ecc.bytes;
1654         int eccsteps = chip->ecc.steps;
1655         const u8 *p;
1656         u8 *ecc_calc = chip->buffers->ecccalc;
1657
1658         debug("%s(buf?%p, oob%d p%x)\n",
1659               __func__, buf, oob_required, page);
1660         for (i = 0; i < chip->ecc.total; i++)
1661                 ecc_calc[i] = 0xFF;
1662
1663         /* Copy the page data from caller's buffers to private buffer */
1664         chip->write_buf(mtd, buf, mtd->writesize);
1665         /* Use private date as source for ECC calculation */
1666         p = tn->buf.dmabuf;
1667
1668         /* Hardware ECC calculation */
1669         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1670                 int ret;
1671
1672                 ret = chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1673
1674                 if (ret < 0)
1675                         debug("calculate(mtd, p?%p, &ecc_calc[%d]?%p) returned %d\n",
1676                               p, i, &ecc_calc[i], ret);
1677
1678                 debug("block offset %lx, ecc offset %x\n", p - buf, i);
1679         }
1680
1681         ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1682                                          chip->ecc.total);
1683         if (ret)
1684                 return ret;
1685
1686         /* Store resulting OOB into private buffer, will be sent to HW */
1687         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1688
1689         return 0;
1690 }
1691
1692 /**
1693  * nand_write_page_raw - [INTERN] raw page write function
1694  * @mtd: mtd info structure
1695  * @chip: nand chip info structure
1696  * @buf: data buffer
1697  * @oob_required: must write chip->oob_poi to OOB
1698  * @page: page number to write
1699  *
1700  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1701  */
1702 static int octeontx_nand_write_page_raw(struct mtd_info *mtd,
1703                                         struct nand_chip *chip,
1704                                         const u8 *buf, int oob_required,
1705                                         int page)
1706 {
1707         chip->write_buf(mtd, buf, mtd->writesize);
1708         if (oob_required)
1709                 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1710
1711         return 0;
1712 }
1713
1714 /**
1715  * octeontx_nand_write_oob_std - [REPLACEABLE] the most common OOB data write
1716  *                             function
1717  * @mtd: mtd info structure
1718  * @chip: nand chip info structure
1719  * @page: page number to write
1720  */
1721 static int octeontx_nand_write_oob_std(struct mtd_info *mtd,
1722                                        struct nand_chip *chip,
1723                                        int page)
1724 {
1725         int status = 0;
1726         const u8 *buf = chip->oob_poi;
1727         int length = mtd->oobsize;
1728
1729         chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
1730         chip->write_buf(mtd, buf, length);
1731         /* Send command to program the OOB data */
1732         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1733
1734         status = chip->waitfunc(mtd, chip);
1735
1736         return status & NAND_STATUS_FAIL ? -EIO : 0;
1737 }
1738
1739 /**
1740  * octeontx_nand_read_page_raw - [INTERN] read raw page data without ecc
1741  * @mtd: mtd info structure
1742  * @chip: nand chip info structure
1743  * @buf: buffer to store read data
1744  * @oob_required: caller requires OOB data read to chip->oob_poi
1745  * @page: page number to read
1746  *
1747  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1748  */
1749 static int octeontx_nand_read_page_raw(struct mtd_info *mtd,
1750                                        struct nand_chip *chip,
1751                                        u8 *buf, int oob_required, int page)
1752 {
1753         chip->read_buf(mtd, buf, mtd->writesize);
1754         if (oob_required)
1755                 chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1756         return 0;
1757 }
1758
1759 static int octeontx_nand_read_oob_std(struct mtd_info *mtd,
1760                                       struct nand_chip *chip,
1761                                       int page)
1762
1763 {
1764         chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
1765         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1766         return 0;
1767 }
1768
1769 static int octeontx_nand_calc_bch_ecc_strength(struct nand_chip *nand)
1770 {
1771         struct mtd_info *mtd = nand_to_mtd(nand);
1772         struct nand_ecc_ctrl *ecc = &nand->ecc;
1773         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1774         int nsteps = mtd->writesize / ecc->size;
1775         int oobchunk = mtd->oobsize / nsteps;
1776
1777         /* ecc->strength determines ecc_level and OOB's ecc_bytes. */
1778         const u8 strengths[]  = {4, 8, 16, 24, 32, 40, 48, 56, 60, 64};
1779         /* first set the desired ecc_level to match strengths[] */
1780         int index = ARRAY_SIZE(strengths) - 1;
1781         int need;
1782
1783         while (index > 0 && !(ecc->options & NAND_ECC_MAXIMIZE) &&
1784                strengths[index - 1] >= ecc->strength)
1785                 index--;
1786
1787         do {
1788                 need = DIV_ROUND_UP(15 * strengths[index], 8);
1789                 if (need <= oobchunk - 2)
1790                         break;
1791         } while (index > 0);
1792
1793         debug("%s: steps ds: %d, strength ds: %d\n", __func__,
1794               nand->ecc_step_ds, nand->ecc_strength_ds);
1795         ecc->strength = strengths[index];
1796         ecc->bytes = need;
1797         debug("%s: strength: %d, bytes: %d\n", __func__, ecc->strength,
1798               ecc->bytes);
1799
1800         if (!tn->eccmask)
1801                 tn->eccmask = devm_kzalloc(tn->dev, ecc->bytes, GFP_KERNEL);
1802         if (!tn->eccmask)
1803                 return -ENOMEM;
1804
1805         return 0;
1806 }
1807
1808 /* sample the BCH signature of an erased (all 0xff) page,
1809  * to XOR into all page traffic, so erased pages have no ECC errors
1810  */
1811 static int octeontx_bch_save_empty_eccmask(struct nand_chip *nand)
1812 {
1813         struct mtd_info *mtd = nand_to_mtd(nand);
1814         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1815         unsigned int eccsize = nand->ecc.size;
1816         unsigned int eccbytes = nand->ecc.bytes;
1817         u8 erased_ecc[eccbytes];
1818         unsigned long erased_handle;
1819         unsigned char *erased_page = dma_alloc_coherent(eccsize,
1820                                                         &erased_handle);
1821         int i;
1822         int rc = 0;
1823
1824         if (!erased_page)
1825                 return -ENOMEM;
1826
1827         memset(erased_page, 0xff, eccsize);
1828         memset(erased_ecc, 0, eccbytes);
1829
1830         rc = octeontx_nand_bch_calculate_ecc_internal(mtd,
1831                                                       (dma_addr_t)erased_handle,
1832                                                       erased_ecc);
1833
1834         free(erased_page);
1835
1836         for (i = 0; i < eccbytes; i++)
1837                 tn->eccmask[i] = erased_ecc[i] ^ 0xff;
1838
1839         return rc;
1840 }
1841
1842 static void octeontx_nfc_chip_sizing(struct nand_chip *nand)
1843 {
1844         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1845         struct mtd_info *mtd = nand_to_mtd(nand);
1846         struct nand_ecc_ctrl *ecc = &nand->ecc;
1847
1848         chip->row_bytes = nand->onfi_params.addr_cycles & 0xf;
1849         chip->col_bytes = nand->onfi_params.addr_cycles >> 4;
1850         debug("%s(%p) row bytes: %d, col bytes: %d, ecc mode: %d\n",
1851               __func__, nand, chip->row_bytes, chip->col_bytes, ecc->mode);
1852
1853         /*
1854          * HW_BCH using OcteonTX BCH engine, or SOFT_BCH laid out in
1855          * HW_BCH-compatible fashion, depending on devtree advice
1856          * and kernel config.
1857          * BCH/NFC hardware capable of subpage ops, not implemented.
1858          */
1859         mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
1860         nand->options |= NAND_NO_SUBPAGE_WRITE;
1861         debug("%s: start steps: %d, size: %d, bytes: %d\n",
1862               __func__, ecc->steps, ecc->size, ecc->bytes);
1863         debug("%s: step ds: %d, strength ds: %d\n", __func__,
1864               nand->ecc_step_ds, nand->ecc_strength_ds);
1865
1866         if (ecc->mode != NAND_ECC_NONE) {
1867                 int nsteps = ecc->steps ? ecc->steps : 1;
1868
1869                 if (ecc->size && ecc->size != mtd->writesize)
1870                         nsteps = mtd->writesize / ecc->size;
1871                 else if (mtd->writesize > def_ecc_size &&
1872                          !(mtd->writesize & (def_ecc_size - 1)))
1873                         nsteps = mtd->writesize / def_ecc_size;
1874                 ecc->steps = nsteps;
1875                 ecc->size = mtd->writesize / nsteps;
1876                 ecc->bytes = mtd->oobsize / nsteps;
1877
1878                 if (nand->ecc_strength_ds)
1879                         ecc->strength = nand->ecc_strength_ds;
1880                 if (nand->ecc_step_ds)
1881                         ecc->size = nand->ecc_step_ds;
1882                 /*
1883                  * no subpage ops, but set subpage-shift to match ecc->steps
1884                  * so mtd_nandbiterrs tests appropriate boundaries
1885                  */
1886                 if (!mtd->subpage_sft && !(ecc->steps & (ecc->steps - 1)))
1887                         mtd->subpage_sft = fls(ecc->steps) - 1;
1888
1889                 if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
1890                         debug("%s: ecc mode: %d\n", __func__, ecc->mode);
1891                         if (ecc->mode != NAND_ECC_SOFT &&
1892                             !octeontx_nand_calc_bch_ecc_strength(nand)) {
1893                                 struct octeontx_nfc *tn =
1894                                         to_otx_nfc(nand->controller);
1895
1896                                 debug("Using hardware BCH engine support\n");
1897                                 ecc->mode = NAND_ECC_HW_SYNDROME;
1898                                 ecc->read_page = octeontx_nand_hw_bch_read_page;
1899                                 ecc->write_page =
1900                                         octeontx_nand_hw_bch_write_page;
1901                                 ecc->read_page_raw =
1902                                         octeontx_nand_read_page_raw;
1903                                 ecc->write_page_raw =
1904                                         octeontx_nand_write_page_raw;
1905                                 ecc->read_oob = octeontx_nand_read_oob_std;
1906                                 ecc->write_oob = octeontx_nand_write_oob_std;
1907
1908                                 ecc->calculate = octeontx_nand_bch_calculate;
1909                                 ecc->correct = octeontx_nand_bch_correct;
1910                                 ecc->hwctl = octeontx_nand_bch_hwctl;
1911
1912                                 debug("NAND chip %d using hw_bch\n",
1913                                       tn->selected_chip);
1914                                 debug(" %d bytes ECC per %d byte block\n",
1915                                       ecc->bytes, ecc->size);
1916                                 debug(" for %d bits of correction per block.",
1917                                       ecc->strength);
1918                                 octeontx_nand_calc_ecc_layout(nand);
1919                                 octeontx_bch_save_empty_eccmask(nand);
1920                         }
1921                 }
1922         }
1923 }
1924
1925 static int octeontx_nfc_chip_init(struct octeontx_nfc *tn, struct udevice *dev,
1926                                   ofnode node)
1927 {
1928         struct octeontx_nand_chip *chip;
1929         struct nand_chip *nand;
1930         struct mtd_info *mtd;
1931         int ret;
1932
1933         chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
1934         if (!chip)
1935                 return -ENOMEM;
1936
1937         debug("%s: Getting chip select\n", __func__);
1938         ret = ofnode_read_s32(node, "reg", &chip->cs);
1939         if (ret) {
1940                 dev_err(dev, "could not retrieve reg property: %d\n", ret);
1941                 return ret;
1942         }
1943
1944         if (chip->cs >= NAND_MAX_CHIPS) {
1945                 dev_err(dev, "invalid reg value: %u (max CS = 7)\n", chip->cs);
1946                 return -EINVAL;
1947         }
1948         debug("%s: chip select: %d\n", __func__, chip->cs);
1949         nand = &chip->nand;
1950         nand->controller = &tn->controller;
1951         if (!tn->controller.active)
1952                 tn->controller.active = nand;
1953
1954         debug("%s: Setting flash node\n", __func__);
1955         nand_set_flash_node(nand, node);
1956
1957         nand->options = 0;
1958         nand->select_chip = octeontx_nand_select_chip;
1959         nand->cmdfunc = octeontx_nand_cmdfunc;
1960         nand->waitfunc = octeontx_nand_waitfunc;
1961         nand->read_byte = octeontx_nand_read_byte;
1962         nand->read_buf = octeontx_nand_read_buf;
1963         nand->write_buf = octeontx_nand_write_buf;
1964         nand->onfi_set_features = octeontx_nand_set_features;
1965         nand->onfi_get_features = octeontx_nand_get_features;
1966         nand->setup_data_interface = octeontx_nand_setup_dat_intf;
1967
1968         mtd = nand_to_mtd(nand);
1969         debug("%s: mtd: %p\n", __func__, mtd);
1970         mtd->dev->parent = dev;
1971
1972         debug("%s: NDF_MISC: 0x%llx\n", __func__,
1973               readq(tn->base + NDF_MISC));
1974
1975         /* TODO: support more then 1 chip */
1976         debug("%s: Scanning identification\n", __func__);
1977         ret = nand_scan_ident(mtd, 1, NULL);
1978         if (ret)
1979                 return ret;
1980
1981         debug("%s: Sizing chip\n", __func__);
1982         octeontx_nfc_chip_sizing(nand);
1983
1984         debug("%s: Scanning tail\n", __func__);
1985         ret = nand_scan_tail(mtd);
1986         if (ret) {
1987                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
1988                 return ret;
1989         }
1990
1991         debug("%s: Registering mtd\n", __func__);
1992         ret = nand_register(0, mtd);
1993
1994         debug("%s: Adding tail\n", __func__);
1995         list_add_tail(&chip->node, &tn->chips);
1996         return 0;
1997 }
1998
1999 static int octeontx_nfc_chips_init(struct octeontx_nfc *tn)
2000 {
2001         struct udevice *dev = tn->dev;
2002         ofnode node = dev->node;
2003         ofnode nand_node;
2004         int nr_chips = of_get_child_count(node);
2005         int ret;
2006
2007         debug("%s: node: %s\n", __func__, ofnode_get_name(node));
2008         debug("%s: %d chips\n", __func__, nr_chips);
2009         if (nr_chips > NAND_MAX_CHIPS) {
2010                 dev_err(dev, "too many NAND chips: %d\n", nr_chips);
2011                 return -EINVAL;
2012         }
2013
2014         if (!nr_chips) {
2015                 debug("no DT NAND chips found\n");
2016                 return -ENODEV;
2017         }
2018
2019         pr_info("%s: scanning %d chips DTs\n", __func__, nr_chips);
2020
2021         ofnode_for_each_subnode(nand_node, node) {
2022                 debug("%s: Calling octeontx_nfc_chip_init(%p, %s, %ld)\n",
2023                       __func__, tn, dev->name, nand_node.of_offset);
2024                 ret = octeontx_nfc_chip_init(tn, dev, nand_node);
2025                 if (ret)
2026                         return ret;
2027         }
2028         return 0;
2029 }
2030
2031 /* Reset NFC and initialize registers. */
2032 static int octeontx_nfc_init(struct octeontx_nfc *tn)
2033 {
2034         const struct nand_sdr_timings *timings;
2035         u64 ndf_misc;
2036         int rc;
2037
2038         /* Initialize values and reset the fifo */
2039         ndf_misc = readq(tn->base + NDF_MISC);
2040
2041         ndf_misc &= ~NDF_MISC_EX_DIS;
2042         ndf_misc |= (NDF_MISC_BT_DIS | NDF_MISC_RST_FF);
2043         writeq(ndf_misc, tn->base + NDF_MISC);
2044         debug("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
2045
2046         /* Bring the fifo out of reset */
2047         ndf_misc &= ~(NDF_MISC_RST_FF);
2048
2049         /* Maximum of co-processor cycles for glitch filtering */
2050         ndf_misc |= FIELD_PREP(NDF_MISC_WAIT_CNT, 0x3f);
2051
2052         writeq(ndf_misc, tn->base + NDF_MISC);
2053
2054         /* Set timing parameters to onfi mode 0 for probing */
2055         timings = onfi_async_timing_mode_to_sdr_timings(0);
2056         if (IS_ERR(timings))
2057                 return PTR_ERR(timings);
2058         rc = set_default_timings(tn, timings);
2059         if (rc)
2060                 return rc;
2061
2062         return 0;
2063 }
2064
2065 static int octeontx_pci_nand_probe(struct udevice *dev)
2066 {
2067         struct octeontx_nfc *tn = dev_get_priv(dev);
2068         int ret;
2069         static bool probe_done;
2070
2071         debug("%s(%s) tn: %p\n", __func__, dev->name, tn);
2072         if (probe_done)
2073                 return 0;
2074
2075         if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
2076                 bch_vf = octeontx_bch_getv();
2077                 if (!bch_vf) {
2078                         struct octeontx_probe_device *probe_dev;
2079
2080                         debug("%s: bch not yet initialized\n", __func__);
2081                         probe_dev = calloc(sizeof(*probe_dev), 1);
2082                         if (!probe_dev) {
2083                                 printf("%s: Out of memory\n", __func__);
2084                                 return -ENOMEM;
2085                         }
2086                         probe_dev->dev = dev;
2087                         INIT_LIST_HEAD(&probe_dev->list);
2088                         list_add_tail(&probe_dev->list,
2089                                       &octeontx_pci_nand_deferred_devices);
2090                         debug("%s: Defering probe until after BCH initialization\n",
2091                               __func__);
2092                         return 0;
2093                 }
2094         }
2095
2096         tn->dev = dev;
2097         INIT_LIST_HEAD(&tn->chips);
2098
2099         tn->base = dm_pci_map_bar(dev, PCI_BASE_ADDRESS_0, PCI_REGION_MEM);
2100         if (!tn->base) {
2101                 ret = -EINVAL;
2102                 goto release;
2103         }
2104         debug("%s: bar at %p\n", __func__, tn->base);
2105         tn->buf.dmabuflen = NAND_MAX_PAGESIZE + NAND_MAX_OOBSIZE;
2106         tn->buf.dmabuf = dma_alloc_coherent(tn->buf.dmabuflen,
2107                                             (unsigned long *)&tn->buf.dmaaddr);
2108         if (!tn->buf.dmabuf) {
2109                 ret = -ENOMEM;
2110                 debug("%s: Could not allocate DMA buffer\n", __func__);
2111                 goto unclk;
2112         }
2113
2114         /* one hw-bch response, for one outstanding transaction */
2115         tn->bch_resp = dma_alloc_coherent(sizeof(*tn->bch_resp),
2116                                           (unsigned long *)&tn->bch_rhandle);
2117
2118         tn->stat = dma_alloc_coherent(8, (unsigned long *)&tn->stat_addr);
2119         if (!tn->stat || !tn->bch_resp) {
2120                 debug("%s: Could not allocate bch status or response\n",
2121                       __func__);
2122                 ret = -ENOMEM;
2123                 goto unclk;
2124         }
2125
2126         debug("%s: Calling octeontx_nfc_init()\n", __func__);
2127         octeontx_nfc_init(tn);
2128         debug("%s: Initializing chips\n", __func__);
2129         ret = octeontx_nfc_chips_init(tn);
2130         debug("%s: init chips ret: %d\n", __func__, ret);
2131         if (ret) {
2132                 if (ret != -ENODEV)
2133                         dev_err(dev, "failed to init nand chips\n");
2134                 goto unclk;
2135         }
2136         dev_info(dev, "probed\n");
2137         return 0;
2138
2139 unclk:
2140 release:
2141         return ret;
2142 }
2143
2144 int octeontx_pci_nand_disable(struct udevice *dev)
2145 {
2146         struct octeontx_nfc *tn = dev_get_priv(dev);
2147         u64 dma_cfg;
2148         u64 ndf_misc;
2149
2150         debug("%s: Disabling NAND device %s\n", __func__, dev->name);
2151         dma_cfg = readq(tn->base + NDF_DMA_CFG);
2152         dma_cfg &= ~NDF_DMA_CFG_EN;
2153         dma_cfg |= NDF_DMA_CFG_CLR;
2154         writeq(dma_cfg, tn->base + NDF_DMA_CFG);
2155
2156         /* Disable execution and put FIFO in reset mode */
2157         ndf_misc = readq(tn->base + NDF_MISC);
2158         ndf_misc |= NDF_MISC_EX_DIS | NDF_MISC_RST_FF;
2159         writeq(ndf_misc, tn->base + NDF_MISC);
2160         ndf_misc &= ~NDF_MISC_RST_FF;
2161         writeq(ndf_misc, tn->base + NDF_MISC);
2162 #ifdef DEBUG
2163         printf("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
2164 #endif
2165         /* Clear any interrupts and enable bits */
2166         writeq(~0ull, tn->base + NDF_INT_ENA_W1C);
2167         writeq(~0ull, tn->base + NDF_INT);
2168         debug("%s: NDF_ST_REG: 0x%llx\n", __func__,
2169               readq(tn->base + NDF_ST_REG));
2170         return 0;
2171 }
2172
2173 /**
2174  * Since it's possible (and even likely) that the NAND device will be probed
2175  * before the BCH device has been probed, we may need to defer the probing.
2176  *
2177  * In this case, the initial probe returns success but the actual probing
2178  * is deferred until the BCH VF has been probed.
2179  *
2180  * @return      0 for success, otherwise error
2181  */
2182 int octeontx_pci_nand_deferred_probe(void)
2183 {
2184         int rc = 0;
2185         struct octeontx_probe_device *pdev;
2186
2187         debug("%s: Performing deferred probing\n", __func__);
2188         list_for_each_entry(pdev, &octeontx_pci_nand_deferred_devices, list) {
2189                 debug("%s: Probing %s\n", __func__, pdev->dev->name);
2190                 pdev->dev->flags &= ~DM_FLAG_ACTIVATED;
2191                 rc = device_probe(pdev->dev);
2192                 if (rc && rc != -ENODEV) {
2193                         printf("%s: Error %d with deferred probe of %s\n",
2194                                __func__, rc, pdev->dev->name);
2195                         break;
2196                 }
2197         }
2198         return rc;
2199 }
2200
2201 static const struct pci_device_id octeontx_nfc_pci_id_table[] = {
2202         { PCI_VDEVICE(CAVIUM, 0xA04F) },
2203         {}
2204 };
2205
2206 static int octeontx_nand_ofdata_to_platdata(struct udevice *dev)
2207 {
2208         return 0;
2209 }
2210
2211 static const struct udevice_id octeontx_nand_ids[] = {
2212         { .compatible = "cavium,cn8130-nand" },
2213         { },
2214 };
2215
2216 U_BOOT_DRIVER(octeontx_pci_nand) = {
2217         .name   = OCTEONTX_NAND_DRIVER_NAME,
2218         .id     = UCLASS_MTD,
2219         .of_match = of_match_ptr(octeontx_nand_ids),
2220         .ofdata_to_platdata = octeontx_nand_ofdata_to_platdata,
2221         .probe = octeontx_pci_nand_probe,
2222         .priv_auto      = sizeof(struct octeontx_nfc),
2223         .remove = octeontx_pci_nand_disable,
2224         .flags = DM_FLAG_OS_PREPARE,
2225 };
2226
2227 U_BOOT_PCI_DEVICE(octeontx_pci_nand, octeontx_nfc_pci_id_table);
2228
2229 void board_nand_init(void)
2230 {
2231         struct udevice *dev;
2232         int ret;
2233
2234         if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
2235                 ret = uclass_get_device_by_driver(UCLASS_MISC,
2236                                                   DM_GET_DRIVER(octeontx_pci_bchpf),
2237                                                   &dev);
2238                 if (ret && ret != -ENODEV) {
2239                         pr_err("Failed to initialize OcteonTX BCH PF controller. (error %d)\n",
2240                                ret);
2241                 }
2242                 ret = uclass_get_device_by_driver(UCLASS_MISC,
2243                                                   DM_GET_DRIVER(octeontx_pci_bchvf),
2244                                                   &dev);
2245                 if (ret && ret != -ENODEV) {
2246                         pr_err("Failed to initialize OcteonTX BCH VF controller. (error %d)\n",
2247                                ret);
2248                 }
2249         }
2250
2251         ret = uclass_get_device_by_driver(UCLASS_MTD,
2252                                           DM_GET_DRIVER(octeontx_pci_nand),
2253                                           &dev);
2254         if (ret && ret != -ENODEV)
2255                 pr_err("Failed to initialize OcteonTX NAND controller. (error %d)\n",
2256                        ret);
2257 }