drivers/mtd/nand/raw/octeontx_nand.c

   1 // SPDX-License-Identifier:    GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Marvell International Ltd.
   4  */
   5
   6 #include <dm.h>
   7 #include <dm/device-internal.h>
   8 #include <dm/devres.h>
   9 #include <dm/of_access.h>
  10 #include <malloc.h>
  11 #include <memalign.h>
  12 #include <nand.h>
  13 #include <pci.h>
  14 #include <time.h>
  15 #include <linux/bitfield.h>
  16 #include <linux/ctype.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/delay.h>
  19 #include <linux/errno.h>
  20 #include <linux/err.h>
  21 #include <linux/ioport.h>
  22 #include <linux/libfdt.h>
  23 #include <linux/mtd/mtd.h>
  24 #include <linux/mtd/nand_bch.h>
  25 #include <linux/mtd/nand_ecc.h>
  26 #include <linux/mtd/rawnand.h>
  27 #include <asm/global_data.h>
  28 #include <asm/io.h>
  29 #include <asm/types.h>
  30 #include <asm/dma-mapping.h>
  31 #include <asm/arch/clock.h>
  32 #include "octeontx_bch.h"
  33
  34 #ifdef DEBUG
  35 # undef CONFIG_LOGLEVEL
  36 # define CONFIG_LOGLEVEL 8
  37 #endif
  38
  39 /*
  40  * The NDF_CMD queue takes commands between 16 - 128 bit.
  41  * All commands must be 16 bit aligned and are little endian.
  42  * WAIT_STATUS commands must be 64 bit aligned.
  43  * Commands are selected by the 4 bit opcode.
  44  *
  45  * Available Commands:
  46  *
  47  * 16 Bit:
  48  *   NOP
  49  *   WAIT
  50  *   BUS_ACQ, BUS_REL
  51  *   CHIP_EN, CHIP_DIS
  52  *
  53  * 32 Bit:
  54  *   CLE_CMD
  55  *   RD_CMD, RD_EDO_CMD
  56  *   WR_CMD
  57  *
  58  * 64 Bit:
  59  *   SET_TM_PAR
  60  *
  61  * 96 Bit:
  62  *   ALE_CMD
  63  *
  64  * 128 Bit:
  65  *   WAIT_STATUS, WAIT_STATUS_ALE
  66  */
  67
  68 /* NDF Register offsets */
  69 #define NDF_CMD                 0x0
  70 #define NDF_MISC                0x8
  71 #define NDF_ECC_CNT             0x10
  72 #define NDF_DRBELL              0x30
  73 #define NDF_ST_REG              0x38    /* status */
  74 #define NDF_INT                 0x40
  75 #define NDF_INT_W1S             0x48
  76 #define NDF_DMA_CFG             0x50
  77 #define NDF_DMA_ADR             0x58
  78 #define NDF_INT_ENA_W1C         0x60
  79 #define NDF_INT_ENA_W1S         0x68
  80
  81 /* NDF command opcodes */
  82 #define NDF_OP_NOP              0x0
  83 #define NDF_OP_SET_TM_PAR       0x1
  84 #define NDF_OP_WAIT             0x2
  85 #define NDF_OP_CHIP_EN_DIS      0x3
  86 #define NDF_OP_CLE_CMD          0x4
  87 #define NDF_OP_ALE_CMD          0x5
  88 #define NDF_OP_WR_CMD           0x8
  89 #define NDF_OP_RD_CMD           0x9
  90 #define NDF_OP_RD_EDO_CMD       0xa
  91 #define NDF_OP_WAIT_STATUS      0xb     /* same opcode for WAIT_STATUS_ALE */
  92 #define NDF_OP_BUS_ACQ_REL      0xf
  93
  94 #define NDF_BUS_ACQUIRE         1
  95 #define NDF_BUS_RELEASE         0
  96
  97 #define DBGX_EDSCR(X)           (0x87A008000088 + (X) * 0x80000)
  98
  99 struct ndf_nop_cmd {
 100         u16 opcode:     4;
 101         u16 nop:        12;
 102 };
 103
 104 struct ndf_wait_cmd {
 105         u16 opcode:4;
 106         u16 r_b:1;              /* wait for one cycle or PBUS_WAIT deassert */
 107         u16:3;
 108         u16 wlen:3;             /* timing parameter select */
 109         u16:5;
 110 };
 111
 112 struct ndf_bus_cmd {
 113         u16 opcode:4;
 114         u16 direction:4;        /* 1 = acquire, 0 = release */
 115         u16:8;
 116 };
 117
 118 struct ndf_chip_cmd {
 119         u16 opcode:4;
 120         u16 chip:3;             /* select chip, 0 = disable */
 121         u16 enable:1;           /* 1 = enable, 0 = disable */
 122         u16 bus_width:2;        /* 10 = 16 bit, 01 = 8 bit */
 123         u16:6;
 124 };
 125
 126 struct ndf_cle_cmd {
 127         u32 opcode:4;
 128         u32:4;
 129         u32 cmd_data:8;         /* command sent to the PBUS AD pins */
 130         u32 clen1:3;            /* time between PBUS CLE and WE asserts */
 131         u32 clen2:3;            /* time WE remains asserted */
 132         u32 clen3:3;            /* time between WE deassert and CLE */
 133         u32:7;
 134 };
 135
 136 /* RD_EDO_CMD uses the same layout as RD_CMD */
 137 struct ndf_rd_cmd {
 138         u32 opcode:4;
 139         u32 data:16;            /* data bytes */
 140         u32 rlen1:3;
 141         u32 rlen2:3;
 142         u32 rlen3:3;
 143         u32 rlen4:3;
 144 };
 145
 146 struct ndf_wr_cmd {
 147         u32 opcode:4;
 148         u32 data:16;            /* data bytes */
 149         u32:4;
 150         u32 wlen1:3;
 151         u32 wlen2:3;
 152         u32:3;
 153 };
 154
 155 struct ndf_set_tm_par_cmd {
 156         u64 opcode:4;
 157         u64 tim_mult:4; /* multiplier for the seven parameters */
 158         u64 tm_par1:8;  /* --> Following are the 7 timing parameters that */
 159         u64 tm_par2:8;  /*     specify the number of coprocessor cycles.  */
 160         u64 tm_par3:8;  /*     A value of zero means one cycle.           */
 161         u64 tm_par4:8;  /*     All values are scaled by tim_mult          */
 162         u64 tm_par5:8;  /*     using tim_par * (2 ^ tim_mult).            */
 163         u64 tm_par6:8;
 164         u64 tm_par7:8;
 165 };
 166
 167 struct ndf_ale_cmd {
 168         u32 opcode:4;
 169         u32:4;
 170         u32 adr_byte_num:4;     /* number of address bytes to be sent */
 171         u32:4;
 172         u32 alen1:3;
 173         u32 alen2:3;
 174         u32 alen3:3;
 175         u32 alen4:3;
 176         u32:4;
 177         u8 adr_byt1;
 178         u8 adr_byt2;
 179         u8 adr_byt3;
 180         u8 adr_byt4;
 181         u8 adr_byt5;
 182         u8 adr_byt6;
 183         u8 adr_byt7;
 184         u8 adr_byt8;
 185 };
 186
 187 struct ndf_wait_status_cmd {
 188         u32 opcode:4;
 189         u32:4;
 190         u32 data:8;             /** data */
 191         u32 clen1:3;
 192         u32 clen2:3;
 193         u32 clen3:3;
 194         u32:8;
 195         /** set to 5 to select WAIT_STATUS_ALE command */
 196         u32 ale_ind:8;
 197         /** ALE only: number of address bytes to be sent */
 198         u32 adr_byte_num:4;
 199         u32:4;
 200         u32 alen1:3;    /* ALE only */
 201         u32 alen2:3;    /* ALE only */
 202         u32 alen3:3;    /* ALE only */
 203         u32 alen4:3;    /* ALE only */
 204         u32:4;
 205         u8 adr_byt[4];          /* ALE only */
 206         u32 nine:4;     /* set to 9 */
 207         u32 and_mask:8;
 208         u32 comp_byte:8;
 209         u32 rlen1:3;
 210         u32 rlen2:3;
 211         u32 rlen3:3;
 212         u32 rlen4:3;
 213 };
 214
 215 union ndf_cmd {
 216         u64 val[2];
 217         union {
 218                 struct ndf_nop_cmd              nop;
 219                 struct ndf_wait_cmd             wait;
 220                 struct ndf_bus_cmd              bus_acq_rel;
 221                 struct ndf_chip_cmd             chip_en_dis;
 222                 struct ndf_cle_cmd              cle_cmd;
 223                 struct ndf_rd_cmd               rd_cmd;
 224                 struct ndf_wr_cmd               wr_cmd;
 225                 struct ndf_set_tm_par_cmd       set_tm_par;
 226                 struct ndf_ale_cmd              ale_cmd;
 227                 struct ndf_wait_status_cmd      wait_status;
 228         } u;
 229 };
 230
 231 /** Disable multi-bit error hangs */
 232 #define NDF_MISC_MB_DIS         BIT_ULL(27)
 233 /** High watermark for NBR FIFO or load/store operations */
 234 #define NDF_MISC_NBR_HWM        GENMASK_ULL(26, 24)
 235 /** Wait input filter count */
 236 #define NDF_MISC_WAIT_CNT       GENMASK_ULL(23, 18)
 237 /** Unfilled NFD_CMD queue bytes */
 238 #define NDF_MISC_FR_BYTE        GENMASK_ULL(17, 7)
 239 /** Set by HW when it reads the last 8 bytes of NDF_CMD */
 240 #define NDF_MISC_RD_DONE        BIT_ULL(6)
 241 /** Set by HW when it reads. SW read of NDF_CMD clears it */
 242 #define NDF_MISC_RD_VAL         BIT_ULL(5)
 243 /** Let HW read NDF_CMD queue. Cleared on SW NDF_CMD write */
 244 #define NDF_MISC_RD_CMD         BIT_ULL(4)
 245 /** Boot disable */
 246 #define NDF_MISC_BT_DIS         BIT_ULL(2)
 247 /** Stop command execution after completing command queue */
 248 #define NDF_MISC_EX_DIS         BIT_ULL(1)
 249 /** Reset fifo */
 250 #define NDF_MISC_RST_FF         BIT_ULL(0)
 251
 252 /** DMA engine enable */
 253 #define NDF_DMA_CFG_EN          BIT_ULL(63)
 254 /** Read or write */
 255 #define NDF_DMA_CFG_RW          BIT_ULL(62)
 256 /** Terminates DMA and clears enable bit */
 257 #define NDF_DMA_CFG_CLR         BIT_ULL(61)
 258 /** 32-bit swap enable */
 259 #define NDF_DMA_CFG_SWAP32      BIT_ULL(59)
 260 /** 16-bit swap enable */
 261 #define NDF_DMA_CFG_SWAP16      BIT_ULL(58)
 262 /** 8-bit swap enable */
 263 #define NDF_DMA_CFG_SWAP8       BIT_ULL(57)
 264 /** Endian mode */
 265 #define NDF_DMA_CFG_CMD_BE      BIT_ULL(56)
 266 /** Number of 64 bit transfers */
 267 #define NDF_DMA_CFG_SIZE        GENMASK_ULL(55, 36)
 268
 269 /** Command execution status idle */
 270 #define NDF_ST_REG_EXE_IDLE     BIT_ULL(15)
 271 /** Command execution SM states */
 272 #define NDF_ST_REG_EXE_SM       GENMASK_ULL(14, 11)
 273 /** DMA and load SM states */
 274 #define NDF_ST_REG_BT_SM        GENMASK_ULL(10, 7)
 275 /** Queue read-back SM bad state */
 276 #define NDF_ST_REG_RD_FF_BAD    BIT_ULL(6)
 277 /** Queue read-back SM states */
 278 #define NDF_ST_REG_RD_FF        GENMASK_ULL(5, 4)
 279 /** Main SM is in a bad state */
 280 #define NDF_ST_REG_MAIN_BAD     BIT_ULL(3)
 281 /** Main SM states */
 282 #define NDF_ST_REG_MAIN_SM      GENMASK_ULL(2, 0)
 283
 284 #define MAX_NAND_NAME_LEN       64
 285 #if (defined(NAND_MAX_PAGESIZE) && (NAND_MAX_PAGESIZE > 4096)) ||       \
 286         !defined(NAND_MAX_PAGESIZE)
 287 # undef NAND_MAX_PAGESIZE
 288 # define NAND_MAX_PAGESIZE      4096
 289 #endif
 290 #if (defined(NAND_MAX_OOBSIZE) && (NAND_MAX_OOBSIZE > 256)) ||          \
 291         !defined(NAND_MAX_OOBSIZE)
 292 # undef NAND_MAX_OOBSIZE
 293 # define NAND_MAX_OOBSIZE       256
 294 #endif
 295
 296 #define OCTEONTX_NAND_DRIVER_NAME       "octeontx_nand"
 297
 298 #define NDF_TIMEOUT             1000    /** Timeout in ms */
 299 #define USEC_PER_SEC            1000000 /** Linux compatibility */
 300 #ifndef NAND_MAX_CHIPS
 301 # define NAND_MAX_CHIPS         8       /** Linux compatibility */
 302 #endif
 303
 304 struct octeontx_nand_chip {
 305         struct list_head node;
 306         struct nand_chip nand;
 307         struct ndf_set_tm_par_cmd timings;
 308         int cs;
 309         int selected_page;
 310         int iface_mode;
 311         int row_bytes;
 312         int col_bytes;
 313         bool oob_only;
 314         bool iface_set;
 315 };
 316
 317 struct octeontx_nand_buf {
 318         u8 *dmabuf;
 319         dma_addr_t dmaaddr;
 320         int dmabuflen;
 321         int data_len;
 322         int data_index;
 323 };
 324
 325 /** NAND flash controller (NDF) related information */
 326 struct octeontx_nfc {
 327         struct nand_hw_control controller;
 328         struct udevice *dev;
 329         void __iomem *base;
 330         struct list_head chips;
 331         int selected_chip;      /* Currently selected NAND chip number */
 332
 333         /*
 334          * Status is separate from octeontx_nand_buf because
 335          * it can be used in parallel and during init.
 336          */
 337         u8 *stat;
 338         dma_addr_t stat_addr;
 339         bool use_status;
 340
 341         struct octeontx_nand_buf buf;
 342         union bch_resp *bch_resp;
 343         dma_addr_t bch_rhandle;
 344
 345         /* BCH of all-0xff, so erased pages read as error-free */
 346         unsigned char *eccmask;
 347 };
 348
 349 /* settable timings - 0..7 select timing of alen1..4/clen1..3/etc */
 350 enum tm_idx {
 351         t0, /* fixed at 4<<mult cycles */
 352         t1, t2, t3, t4, t5, t6, t7, /* settable per ONFI-timing mode */
 353 };
 354
 355 struct octeontx_probe_device {
 356         struct list_head list;
 357         struct udevice *dev;
 358 };
 359
 360 static struct bch_vf *bch_vf;
 361 /** Deferred devices due to BCH not being ready */
 362 LIST_HEAD(octeontx_pci_nand_deferred_devices);
 363
 364 /** default parameters used for probing chips */
 365 #define MAX_ONFI_MODE   5
 366
 367 static int default_onfi_timing;
 368 static int slew_ns = 2; /* default timing padding */
 369 static int def_ecc_size = 512; /* 1024 best for sw_bch, <= 4095 for hw_bch */
 370 static int default_width = 1; /* 8 bit */
 371 static int default_page_size = 2048;
 372 static struct ndf_set_tm_par_cmd default_timing_parms;
 373
 374 /** Port from Linux */
 375 #define readq_poll_timeout(addr, val, cond, delay_us, timeout_us)       \
 376 ({                                                                      \
 377         ulong __start = get_timer(0);                                   \
 378         void *__addr = (addr);                                          \
 379         const ulong __timeout_ms = timeout_us / 1000;                   \
 380         do {                                                            \
 381                 (val) = readq(__addr);                                  \
 382                 if (cond)                                               \
 383                         break;                                          \
 384                 if (timeout_us && get_timer(__start) > __timeout_ms) {  \
 385                         (val) = readq(__addr);                          \
 386                         break;                                          \
 387                 }                                                       \
 388                 if (delay_us)                                           \
 389                         udelay(delay_us);                               \
 390         } while (1);                                                    \
 391         (cond) ? 0 : -ETIMEDOUT;                                        \
 392 })
 393
 394 /** Ported from Linux 4.9.0 include/linux/of.h for compatibility */
 395 static inline int of_get_child_count(const ofnode node)
 396 {
 397         return fdtdec_get_child_count(gd->fdt_blob, ofnode_to_offset(node));
 398 }
 399
 400 /**
 401  * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
 402  */
 403 static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
 404                                  struct mtd_oob_region *oobregion)
 405 {
 406         struct nand_chip *chip = mtd_to_nand(mtd);
 407         struct nand_ecc_ctrl *ecc = &chip->ecc;
 408
 409         if (section || !ecc->total)
 410                 return -ERANGE;
 411
 412         oobregion->length = ecc->total;
 413         oobregion->offset = mtd->oobsize - oobregion->length;
 414
 415         return 0;
 416 }
 417
 418 /**
 419  * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
 420  */
 421 static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
 422                                   struct mtd_oob_region *oobregion)
 423 {
 424         struct nand_chip *chip = mtd_to_nand(mtd);
 425         struct nand_ecc_ctrl *ecc = &chip->ecc;
 426
 427         if (section)
 428                 return -ERANGE;
 429
 430         oobregion->length = mtd->oobsize - ecc->total - 2;
 431         oobregion->offset = 2;
 432
 433         return 0;
 434 }
 435
 436 static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
 437         .ecc = nand_ooblayout_ecc_lp,
 438         .rfree = nand_ooblayout_free_lp,
 439 };
 440
 441 static inline struct octeontx_nand_chip *to_otx_nand(struct nand_chip *nand)
 442 {
 443         return container_of(nand, struct octeontx_nand_chip, nand);
 444 }
 445
 446 static inline struct octeontx_nfc *to_otx_nfc(struct nand_hw_control *ctrl)
 447 {
 448         return container_of(ctrl, struct octeontx_nfc, controller);
 449 }
 450
 451 static int octeontx_nand_calc_ecc_layout(struct nand_chip *nand)
 452 {
 453         struct nand_ecclayout *layout = nand->ecc.layout;
 454         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 455         struct mtd_info *mtd = &nand->mtd;
 456         int oobsize = mtd->oobsize;
 457         int i;
 458         bool layout_alloc = false;
 459
 460         if (!layout) {
 461                 layout = devm_kzalloc(tn->dev, sizeof(*layout), GFP_KERNEL);
 462                 if (!layout)
 463                         return -ENOMEM;
 464                 nand->ecc.layout = layout;
 465                 layout_alloc = true;
 466         }
 467         layout->eccbytes = nand->ecc.steps * nand->ecc.bytes;
 468         /* Reserve 2 bytes for bad block marker */
 469         if (layout->eccbytes + 2 > oobsize) {
 470                 pr_err("No suitable oob scheme available for oobsize %d eccbytes %u\n",
 471                        oobsize, layout->eccbytes);
 472                 goto fail;
 473         }
 474         /* put ecc bytes at oob tail */
 475         for (i = 0; i < layout->eccbytes; i++)
 476                 layout->eccpos[i] = oobsize - layout->eccbytes + i;
 477         layout->oobfree[0].offset = 2;
 478         layout->oobfree[0].length = oobsize - 2 - layout->eccbytes;
 479         nand->ecc.layout = layout;
 480         return 0;
 481
 482 fail:
 483         if (layout_alloc)
 484                 kfree(layout);
 485         return -1;
 486 }
 487
 488 /*
 489  * Read a single byte from the temporary buffer. Used after READID
 490  * to get the NAND information and for STATUS.
 491  */
 492 static u8 octeontx_nand_read_byte(struct mtd_info *mtd)
 493 {
 494         struct nand_chip *nand = mtd_to_nand(mtd);
 495         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 496
 497         if (tn->use_status) {
 498                 tn->use_status = false;
 499                 return *tn->stat;
 500         }
 501
 502         if (tn->buf.data_index < tn->buf.data_len)
 503                 return tn->buf.dmabuf[tn->buf.data_index++];
 504
 505         dev_err(tn->dev, "No data to read, idx: 0x%x, len: 0x%x\n",
 506                 tn->buf.data_index, tn->buf.data_len);
 507
 508         return 0xff;
 509 }
 510
 511 /*
 512  * Read a number of pending bytes from the temporary buffer. Used
 513  * to get page and OOB data.
 514  */
 515 static void octeontx_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len)
 516 {
 517         struct nand_chip *nand = mtd_to_nand(mtd);
 518         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 519
 520         if (len > tn->buf.data_len - tn->buf.data_index) {
 521                 dev_err(tn->dev, "Not enough data for read of %d bytes\n", len);
 522                 return;
 523         }
 524
 525         memcpy(buf, tn->buf.dmabuf + tn->buf.data_index, len);
 526         tn->buf.data_index += len;
 527 }
 528
 529 static void octeontx_nand_write_buf(struct mtd_info *mtd,
 530                                     const u8 *buf, int len)
 531 {
 532         struct nand_chip *nand = mtd_to_nand(mtd);
 533         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
 534
 535         memcpy(tn->buf.dmabuf + tn->buf.data_len, buf, len);
 536         tn->buf.data_len += len;
 537 }
 538
 539 /* Overwrite default function to avoid sync abort on chip = -1. */
 540 static void octeontx_nand_select_chip(struct mtd_info *mtd, int chip)
 541 {
 542 }
 543
 544 static inline int timing_to_cycle(u32 psec, unsigned long clock)
 545 {
 546         unsigned int ns;
 547         int ticks;
 548
 549         ns = DIV_ROUND_UP(psec, 1000);
 550         ns += slew_ns;
 551
 552         /* no rounding needed since clock is multiple of 1MHz */
 553         clock /= 1000000;
 554         ns *= clock;
 555
 556         ticks = DIV_ROUND_UP(ns, 1000);
 557
 558         /* actual delay is (tm_parX+1)<<tim_mult */
 559         if (ticks)
 560                 ticks--;
 561
 562         return ticks;
 563 }
 564
 565 static void set_timings(struct octeontx_nand_chip *chip,
 566                         struct ndf_set_tm_par_cmd *tp,
 567                         const struct nand_sdr_timings *timings,
 568                         unsigned long sclk)
 569 {
 570         /* scaled coprocessor-cycle values */
 571         u32 s_wh, s_cls, s_clh, s_rp, s_wb, s_wc;
 572
 573         tp->tim_mult = 0;
 574         s_wh = timing_to_cycle(timings->tWH_min, sclk);
 575         s_cls = timing_to_cycle(timings->tCLS_min, sclk);
 576         s_clh = timing_to_cycle(timings->tCLH_min, sclk);
 577         s_rp = timing_to_cycle(timings->tRP_min, sclk);
 578         s_wb = timing_to_cycle(timings->tWB_max, sclk);
 579         s_wc = timing_to_cycle(timings->tWC_min, sclk);
 580
 581         tp->tm_par1 = s_wh;
 582         tp->tm_par2 = s_clh;
 583         tp->tm_par3 = s_rp + 1;
 584         tp->tm_par4 = s_cls - s_wh;
 585         tp->tm_par5 = s_wc - s_wh + 1;
 586         tp->tm_par6 = s_wb;
 587         tp->tm_par7 = 0;
 588         tp->tim_mult++; /* overcompensate for bad math */
 589
 590         /* TODO: comment parameter re-use */
 591
 592         pr_debug("%s: tim_par: mult: %d  p1: %d  p2: %d  p3: %d\n",
 593                  __func__, tp->tim_mult, tp->tm_par1, tp->tm_par2, tp->tm_par3);
 594         pr_debug("                 p4: %d  p5: %d  p6: %d  p7: %d\n",
 595                  tp->tm_par4, tp->tm_par5, tp->tm_par6, tp->tm_par7);
 596 }
 597
 598 static int set_default_timings(struct octeontx_nfc *tn,
 599                                const struct nand_sdr_timings *timings)
 600 {
 601         unsigned long sclk = octeontx_get_io_clock();
 602
 603         set_timings(NULL, &default_timing_parms, timings, sclk);
 604         return 0;
 605 }
 606
 607 static int octeontx_nfc_chip_set_timings(struct octeontx_nand_chip *chip,
 608                                          const struct nand_sdr_timings *timings)
 609 {
 610         /*struct octeontx_nfc *tn = to_otx_nfc(chip->nand.controller);*/
 611         unsigned long sclk = octeontx_get_io_clock();
 612
 613         set_timings(chip, &chip->timings, timings, sclk);
 614         return 0;
 615 }
 616
 617 /* How many bytes are free in the NFD_CMD queue? */
 618 static int ndf_cmd_queue_free(struct octeontx_nfc *tn)
 619 {
 620         u64 ndf_misc;
 621
 622         ndf_misc = readq(tn->base + NDF_MISC);
 623         return FIELD_GET(NDF_MISC_FR_BYTE, ndf_misc);
 624 }
 625
 626 /* Submit a command to the NAND command queue. */
 627 static int ndf_submit(struct octeontx_nfc *tn, union ndf_cmd *cmd)
 628 {
 629         int opcode = cmd->val[0] & 0xf;
 630
 631         switch (opcode) {
 632         /* All these commands fit in one 64bit word */
 633         case NDF_OP_NOP:
 634         case NDF_OP_SET_TM_PAR:
 635         case NDF_OP_WAIT:
 636         case NDF_OP_CHIP_EN_DIS:
 637         case NDF_OP_CLE_CMD:
 638         case NDF_OP_WR_CMD:
 639         case NDF_OP_RD_CMD:
 640         case NDF_OP_RD_EDO_CMD:
 641         case NDF_OP_BUS_ACQ_REL:
 642                 if (ndf_cmd_queue_free(tn) < 8)
 643                         goto full;
 644                 writeq(cmd->val[0], tn->base + NDF_CMD);
 645                 break;
 646         case NDF_OP_ALE_CMD:
 647                 /* ALE commands take either one or two 64bit words */
 648                 if (cmd->u.ale_cmd.adr_byte_num < 5) {
 649                         if (ndf_cmd_queue_free(tn) < 8)
 650                                 goto full;
 651                         writeq(cmd->val[0], tn->base + NDF_CMD);
 652                 } else {
 653                         if (ndf_cmd_queue_free(tn) < 16)
 654                                 goto full;
 655                         writeq(cmd->val[0], tn->base + NDF_CMD);
 656                         writeq(cmd->val[1], tn->base + NDF_CMD);
 657                 }
 658                 break;
 659         case NDF_OP_WAIT_STATUS: /* Wait status commands take two 64bit words */
 660                 if (ndf_cmd_queue_free(tn) < 16)
 661                         goto full;
 662                 writeq(cmd->val[0], tn->base + NDF_CMD);
 663                 writeq(cmd->val[1], tn->base + NDF_CMD);
 664                 break;
 665         default:
 666                 dev_err(tn->dev, "%s: unknown command: %u\n", __func__, opcode);
 667                 return -EINVAL;
 668         }
 669         return 0;
 670
 671 full:
 672         dev_err(tn->dev, "%s: no space left in command queue\n", __func__);
 673         return -ENOMEM;
 674 }
 675
 676 /**
 677  * Wait for the ready/busy signal. First wait for busy to be valid,
 678  * then wait for busy to de-assert.
 679  */
 680 static int ndf_build_wait_busy(struct octeontx_nfc *tn)
 681 {
 682         union ndf_cmd cmd;
 683
 684         memset(&cmd, 0, sizeof(cmd));
 685         cmd.u.wait.opcode = NDF_OP_WAIT;
 686         cmd.u.wait.r_b = 1;
 687         cmd.u.wait.wlen = t6;
 688
 689         if (ndf_submit(tn, &cmd))
 690                 return -ENOMEM;
 691         return 0;
 692 }
 693
 694 static bool ndf_dma_done(struct octeontx_nfc *tn)
 695 {
 696         u64 dma_cfg;
 697
 698         /* Enable bit should be clear after a transfer */
 699         dma_cfg = readq(tn->base + NDF_DMA_CFG);
 700         if (!(dma_cfg & NDF_DMA_CFG_EN))
 701                 return true;
 702
 703         return false;
 704 }
 705
 706 static int ndf_wait(struct octeontx_nfc *tn)
 707 {
 708         ulong start = get_timer(0);
 709         bool done;
 710
 711         while (!(done = ndf_dma_done(tn)) && get_timer(start) < NDF_TIMEOUT)
 712                 ;
 713
 714         if (!done) {
 715                 dev_err(tn->dev, "%s: timeout error\n", __func__);
 716                 return -ETIMEDOUT;
 717         }
 718         return 0;
 719 }
 720
 721 static int ndf_wait_idle(struct octeontx_nfc *tn)
 722 {
 723         u64 val;
 724         u64 dval = 0;
 725         int rc;
 726         int pause = 100;
 727         u64 tot_us = USEC_PER_SEC / 10;
 728
 729         rc = readq_poll_timeout(tn->base + NDF_ST_REG,
 730                                 val, val & NDF_ST_REG_EXE_IDLE, pause, tot_us);
 731         if (!rc)
 732                 rc = readq_poll_timeout(tn->base + NDF_DMA_CFG,
 733                                         dval, !(dval & NDF_DMA_CFG_EN),
 734                                         pause, tot_us);
 735
 736         return rc;
 737 }
 738
 739 /** Issue set timing parameters */
 740 static int ndf_queue_cmd_timing(struct octeontx_nfc *tn,
 741                                 struct ndf_set_tm_par_cmd *timings)
 742 {
 743         union ndf_cmd cmd;
 744
 745         memset(&cmd, 0, sizeof(cmd));
 746         cmd.u.set_tm_par.opcode = NDF_OP_SET_TM_PAR;
 747         cmd.u.set_tm_par.tim_mult = timings->tim_mult;
 748         cmd.u.set_tm_par.tm_par1 = timings->tm_par1;
 749         cmd.u.set_tm_par.tm_par2 = timings->tm_par2;
 750         cmd.u.set_tm_par.tm_par3 = timings->tm_par3;
 751         cmd.u.set_tm_par.tm_par4 = timings->tm_par4;
 752         cmd.u.set_tm_par.tm_par5 = timings->tm_par5;
 753         cmd.u.set_tm_par.tm_par6 = timings->tm_par6;
 754         cmd.u.set_tm_par.tm_par7 = timings->tm_par7;
 755         return ndf_submit(tn, &cmd);
 756 }
 757
 758 /** Issue bus acquire or release */
 759 static int ndf_queue_cmd_bus(struct octeontx_nfc *tn, int direction)
 760 {
 761         union ndf_cmd cmd;
 762
 763         memset(&cmd, 0, sizeof(cmd));
 764         cmd.u.bus_acq_rel.opcode = NDF_OP_BUS_ACQ_REL;
 765         cmd.u.bus_acq_rel.direction = direction;
 766         return ndf_submit(tn, &cmd);
 767 }
 768
 769 /* Issue chip select or deselect */
 770 static int ndf_queue_cmd_chip(struct octeontx_nfc *tn, int enable, int chip,
 771                               int width)
 772 {
 773         union ndf_cmd cmd;
 774
 775         memset(&cmd, 0, sizeof(cmd));
 776         cmd.u.chip_en_dis.opcode = NDF_OP_CHIP_EN_DIS;
 777         cmd.u.chip_en_dis.chip = chip;
 778         cmd.u.chip_en_dis.enable = enable;
 779         cmd.u.chip_en_dis.bus_width = width;
 780         return ndf_submit(tn, &cmd);
 781 }
 782
 783 static int ndf_queue_cmd_wait(struct octeontx_nfc *tn, int t_delay)
 784 {
 785         union ndf_cmd cmd;
 786
 787         memset(&cmd, 0, sizeof(cmd));
 788         cmd.u.wait.opcode = NDF_OP_WAIT;
 789         cmd.u.wait.wlen = t_delay;
 790         return ndf_submit(tn, &cmd);
 791 }
 792
 793 static int ndf_queue_cmd_cle(struct octeontx_nfc *tn, int command)
 794 {
 795         union ndf_cmd cmd;
 796
 797         memset(&cmd, 0, sizeof(cmd));
 798         cmd.u.cle_cmd.opcode = NDF_OP_CLE_CMD;
 799         cmd.u.cle_cmd.cmd_data = command;
 800         cmd.u.cle_cmd.clen1 = t4;
 801         cmd.u.cle_cmd.clen2 = t1;
 802         cmd.u.cle_cmd.clen3 = t2;
 803         return ndf_submit(tn, &cmd);
 804 }
 805
 806 static int ndf_queue_cmd_ale(struct octeontx_nfc *tn, int addr_bytes,
 807                              struct nand_chip *nand, u64 page,
 808                              u32 col, int page_size)
 809 {
 810         struct octeontx_nand_chip *octeontx_nand = (nand) ?
 811                                                 to_otx_nand(nand) : NULL;
 812         union ndf_cmd cmd;
 813
 814         memset(&cmd, 0, sizeof(cmd));
 815         cmd.u.ale_cmd.opcode = NDF_OP_ALE_CMD;
 816         cmd.u.ale_cmd.adr_byte_num = addr_bytes;
 817
 818         /* set column bit for OOB area, assume OOB follows page */
 819         if (octeontx_nand && octeontx_nand->oob_only)
 820                 col += page_size;
 821
 822         /* page is u64 for this generality, even if cmdfunc() passes int */
 823         switch (addr_bytes) {
 824         /* 4-8 bytes: page, then 2-byte col */
 825         case 8:
 826                 cmd.u.ale_cmd.adr_byt8 = (page >> 40) & 0xff;
 827                 fallthrough;
 828         case 7:
 829                 cmd.u.ale_cmd.adr_byt7 = (page >> 32) & 0xff;
 830                 fallthrough;
 831         case 6:
 832                 cmd.u.ale_cmd.adr_byt6 = (page >> 24) & 0xff;
 833                 fallthrough;
 834         case 5:
 835                 cmd.u.ale_cmd.adr_byt5 = (page >> 16) & 0xff;
 836                 fallthrough;
 837         case 4:
 838                 cmd.u.ale_cmd.adr_byt4 = (page >> 8) & 0xff;
 839                 cmd.u.ale_cmd.adr_byt3 = page & 0xff;
 840                 cmd.u.ale_cmd.adr_byt2 = (col >> 8) & 0xff;
 841                 cmd.u.ale_cmd.adr_byt1 =  col & 0xff;
 842                 break;
 843         /* 1-3 bytes: just the page address */
 844         case 3:
 845                 cmd.u.ale_cmd.adr_byt3 = (page >> 16) & 0xff;
 846                 fallthrough;
 847         case 2:
 848                 cmd.u.ale_cmd.adr_byt2 = (page >> 8) & 0xff;
 849                 fallthrough;
 850         case 1:
 851                 cmd.u.ale_cmd.adr_byt1 = page & 0xff;
 852                 break;
 853         default:
 854                 break;
 855         }
 856
 857         cmd.u.ale_cmd.alen1 = t3;
 858         cmd.u.ale_cmd.alen2 = t1;
 859         cmd.u.ale_cmd.alen3 = t5;
 860         cmd.u.ale_cmd.alen4 = t2;
 861         return ndf_submit(tn, &cmd);
 862 }
 863
 864 static int ndf_queue_cmd_write(struct octeontx_nfc *tn, int len)
 865 {
 866         union ndf_cmd cmd;
 867
 868         memset(&cmd, 0, sizeof(cmd));
 869         cmd.u.wr_cmd.opcode = NDF_OP_WR_CMD;
 870         cmd.u.wr_cmd.data = len;
 871         cmd.u.wr_cmd.wlen1 = t3;
 872         cmd.u.wr_cmd.wlen2 = t1;
 873         return ndf_submit(tn, &cmd);
 874 }
 875
 876 static int ndf_build_pre_cmd(struct octeontx_nfc *tn, int cmd1,
 877                              int addr_bytes, u64 page, u32 col, int cmd2)
 878 {
 879         struct nand_chip *nand = tn->controller.active;
 880         struct octeontx_nand_chip *octeontx_nand;
 881         struct ndf_set_tm_par_cmd *timings;
 882         int width, page_size, rc;
 883
 884         /* Also called before chip probing is finished */
 885         if (!nand) {
 886                 timings = &default_timing_parms;
 887                 page_size = default_page_size;
 888                 width = default_width;
 889         } else {
 890                 octeontx_nand = to_otx_nand(nand);
 891                 timings = &octeontx_nand->timings;
 892                 page_size = nand->mtd.writesize;
 893                 if (nand->options & NAND_BUSWIDTH_16)
 894                         width = 2;
 895                 else
 896                         width = 1;
 897         }
 898         rc = ndf_queue_cmd_timing(tn, timings);
 899         if (rc)
 900                 return rc;
 901
 902         rc = ndf_queue_cmd_bus(tn, NDF_BUS_ACQUIRE);
 903         if (rc)
 904                 return rc;
 905
 906         rc = ndf_queue_cmd_chip(tn, 1, tn->selected_chip, width);
 907         if (rc)
 908                 return rc;
 909
 910         rc = ndf_queue_cmd_wait(tn, t1);
 911         if (rc)
 912                 return rc;
 913
 914         rc = ndf_queue_cmd_cle(tn, cmd1);
 915         if (rc)
 916                 return rc;
 917
 918         if (addr_bytes) {
 919                 rc = ndf_build_wait_busy(tn);
 920                 if (rc)
 921                         return rc;
 922
 923                 rc = ndf_queue_cmd_ale(tn, addr_bytes, nand,
 924                                        page, col, page_size);
 925                 if (rc)
 926                         return rc;
 927         }
 928
 929         /* CLE 2 */
 930         if (cmd2) {
 931                 rc = ndf_build_wait_busy(tn);
 932                 if (rc)
 933                         return rc;
 934
 935                 rc = ndf_queue_cmd_cle(tn, cmd2);
 936                 if (rc)
 937                         return rc;
 938         }
 939         return 0;
 940 }
 941
 942 static int ndf_build_post_cmd(struct octeontx_nfc *tn, int hold_time)
 943 {
 944         int rc;
 945
 946         /* Deselect chip */
 947         rc = ndf_queue_cmd_chip(tn, 0, 0, 0);
 948         if (rc)
 949                 return rc;
 950
 951         rc = ndf_queue_cmd_wait(tn, t2);
 952         if (rc)
 953                 return rc;
 954
 955         /* Release bus */
 956         rc = ndf_queue_cmd_bus(tn, 0);
 957         if (rc)
 958                 return rc;
 959
 960         rc = ndf_queue_cmd_wait(tn, hold_time);
 961         if (rc)
 962                 return rc;
 963
 964         /*
 965          * Last action is ringing the doorbell with number of bus
 966          * acquire-releases cycles (currently 1).
 967          */
 968         writeq(1, tn->base + NDF_DRBELL);
 969         return 0;
 970 }
 971
 972 /* Setup the NAND DMA engine for a transfer. */
 973 static void ndf_setup_dma(struct octeontx_nfc *tn, int is_write,
 974                           dma_addr_t bus_addr, int len)
 975 {
 976         u64 dma_cfg;
 977
 978         dma_cfg = FIELD_PREP(NDF_DMA_CFG_RW, is_write) |
 979                   FIELD_PREP(NDF_DMA_CFG_SIZE, (len >> 3) - 1);
 980         dma_cfg |= NDF_DMA_CFG_EN;
 981         writeq(bus_addr, tn->base + NDF_DMA_ADR);
 982         writeq(dma_cfg, tn->base + NDF_DMA_CFG);
 983 }
 984
 985 static int octeontx_nand_reset(struct octeontx_nfc *tn)
 986 {
 987         int rc;
 988
 989         rc = ndf_build_pre_cmd(tn, NAND_CMD_RESET, 0, 0, 0, 0);
 990         if (rc)
 991                 return rc;
 992
 993         rc = ndf_build_wait_busy(tn);
 994         if (rc)
 995                 return rc;
 996
 997         rc = ndf_build_post_cmd(tn, t2);
 998         if (rc)
 999                 return rc;
1000
1001         return 0;
1002 }
1003
1004 static int ndf_read(struct octeontx_nfc *tn, int cmd1, int addr_bytes,
1005                     u64 page, u32 col, int cmd2, int len)
1006 {
1007         dma_addr_t bus_addr = tn->use_status ? tn->stat_addr : tn->buf.dmaaddr;
1008         struct nand_chip *nand = tn->controller.active;
1009         int timing_mode, bytes, rc;
1010         union ndf_cmd cmd;
1011         u64 start, end;
1012
1013         pr_debug("%s(%p, 0x%x, 0x%x, 0x%llx, 0x%x, 0x%x, 0x%x)\n", __func__,
1014                  tn, cmd1, addr_bytes, page, col, cmd2, len);
1015         if (!nand)
1016                 timing_mode = default_onfi_timing;
1017         else
1018                 timing_mode = nand->onfi_timing_mode_default;
1019
1020         /* Build the command and address cycles */
1021         rc = ndf_build_pre_cmd(tn, cmd1, addr_bytes, page, col, cmd2);
1022         if (rc) {
1023                 dev_err(tn->dev, "Build pre command failed\n");
1024                 return rc;
1025         }
1026
1027         /* This waits for some time, then waits for busy to be de-asserted. */
1028         rc = ndf_build_wait_busy(tn);
1029         if (rc) {
1030                 dev_err(tn->dev, "Wait timeout\n");
1031                 return rc;
1032         }
1033
1034         memset(&cmd, 0, sizeof(cmd));
1035
1036         if (timing_mode < 4)
1037                 cmd.u.rd_cmd.opcode = NDF_OP_RD_CMD;
1038         else
1039                 cmd.u.rd_cmd.opcode = NDF_OP_RD_EDO_CMD;
1040
1041         cmd.u.rd_cmd.data = len;
1042         cmd.u.rd_cmd.rlen1 = t7;
1043         cmd.u.rd_cmd.rlen2 = t3;
1044         cmd.u.rd_cmd.rlen3 = t1;
1045         cmd.u.rd_cmd.rlen4 = t7;
1046         rc = ndf_submit(tn, &cmd);
1047         if (rc) {
1048                 dev_err(tn->dev, "Error submitting command\n");
1049                 return rc;
1050         }
1051
1052         start = (u64)bus_addr;
1053         ndf_setup_dma(tn, 0, bus_addr, len);
1054
1055         rc = ndf_build_post_cmd(tn, t2);
1056         if (rc) {
1057                 dev_err(tn->dev, "Build post command failed\n");
1058                 return rc;
1059         }
1060
1061         /* Wait for the DMA to complete */
1062         rc = ndf_wait(tn);
1063         if (rc) {
1064                 dev_err(tn->dev, "DMA timed out\n");
1065                 return rc;
1066         }
1067
1068         end = readq(tn->base + NDF_DMA_ADR);
1069         bytes = end - start;
1070
1071         /* Make sure NDF is really done */
1072         rc = ndf_wait_idle(tn);
1073         if (rc) {
1074                 dev_err(tn->dev, "poll idle failed\n");
1075                 return rc;
1076         }
1077
1078         pr_debug("%s: Read %d bytes\n", __func__, bytes);
1079         return bytes;
1080 }
1081
1082 static int octeontx_nand_get_features(struct mtd_info *mtd,
1083                                       struct nand_chip *chip, int feature_addr,
1084                                       u8 *subfeature_para)
1085 {
1086         struct nand_chip *nand = chip;
1087         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1088         int len = 8;
1089         int rc;
1090
1091         pr_debug("%s: feature addr: 0x%x\n", __func__, feature_addr);
1092         memset(tn->buf.dmabuf, 0xff, len);
1093         tn->buf.data_index = 0;
1094         tn->buf.data_len = 0;
1095         rc = ndf_read(tn, NAND_CMD_GET_FEATURES, 1, feature_addr, 0, 0, len);
1096         if (rc)
1097                 return rc;
1098
1099         memcpy(subfeature_para, tn->buf.dmabuf, ONFI_SUBFEATURE_PARAM_LEN);
1100
1101         return 0;
1102 }
1103
1104 static int octeontx_nand_set_features(struct mtd_info *mtd,
1105                                       struct nand_chip *chip, int feature_addr,
1106                                       u8 *subfeature_para)
1107 {
1108         struct nand_chip *nand = chip;
1109         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1110         const int len = ONFI_SUBFEATURE_PARAM_LEN;
1111         int rc;
1112
1113         rc = ndf_build_pre_cmd(tn, NAND_CMD_SET_FEATURES,
1114                                1, feature_addr, 0, 0);
1115         if (rc)
1116                 return rc;
1117
1118         memcpy(tn->buf.dmabuf, subfeature_para, len);
1119         memset(tn->buf.dmabuf + len, 0, 8 - len);
1120
1121         ndf_setup_dma(tn, 1, tn->buf.dmaaddr, 8);
1122
1123         rc = ndf_queue_cmd_write(tn, 8);
1124         if (rc)
1125                 return rc;
1126
1127         rc = ndf_build_wait_busy(tn);
1128         if (rc)
1129                 return rc;
1130
1131         rc = ndf_build_post_cmd(tn, t2);
1132         if (rc)
1133                 return rc;
1134
1135         return 0;
1136 }
1137
1138 /*
1139  * Read a page from NAND. If the buffer has room, the out of band
1140  * data will be included.
1141  */
1142 static int ndf_page_read(struct octeontx_nfc *tn, u64 page, int col, int len)
1143 {
1144         debug("%s(%p, 0x%llx, 0x%x, 0x%x) active: %p\n", __func__,
1145               tn, page, col, len, tn->controller.active);
1146         struct nand_chip *nand = tn->controller.active;
1147         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1148         int addr_bytes = chip->row_bytes + chip->col_bytes;
1149
1150         memset(tn->buf.dmabuf, 0xff, len);
1151         return ndf_read(tn, NAND_CMD_READ0, addr_bytes,
1152                     page, col, NAND_CMD_READSTART, len);
1153 }
1154
1155 /* Erase a NAND block */
1156 static int ndf_block_erase(struct octeontx_nfc *tn, u64 page_addr)
1157 {
1158         struct nand_chip *nand = tn->controller.active;
1159         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1160         int addr_bytes = chip->row_bytes;
1161         int rc;
1162
1163         rc = ndf_build_pre_cmd(tn, NAND_CMD_ERASE1, addr_bytes,
1164                                page_addr, 0, NAND_CMD_ERASE2);
1165         if (rc)
1166                 return rc;
1167
1168         /* Wait for R_B to signal erase is complete  */
1169         rc = ndf_build_wait_busy(tn);
1170         if (rc)
1171                 return rc;
1172
1173         rc = ndf_build_post_cmd(tn, t2);
1174         if (rc)
1175                 return rc;
1176
1177         /* Wait until the command queue is idle */
1178         return ndf_wait_idle(tn);
1179 }
1180
1181 /*
1182  * Write a page (or less) to NAND.
1183  */
1184 static int ndf_page_write(struct octeontx_nfc *tn, int page)
1185 {
1186         int len, rc;
1187         struct nand_chip *nand = tn->controller.active;
1188         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1189         int addr_bytes = chip->row_bytes + chip->col_bytes;
1190
1191         len = tn->buf.data_len - tn->buf.data_index;
1192         chip->oob_only = (tn->buf.data_index >= nand->mtd.writesize);
1193         WARN_ON_ONCE(len & 0x7);
1194
1195         ndf_setup_dma(tn, 1, tn->buf.dmaaddr + tn->buf.data_index, len);
1196         rc = ndf_build_pre_cmd(tn, NAND_CMD_SEQIN, addr_bytes, page, 0, 0);
1197         if (rc)
1198                 return rc;
1199
1200         rc = ndf_queue_cmd_write(tn, len);
1201         if (rc)
1202                 return rc;
1203
1204         rc = ndf_queue_cmd_cle(tn, NAND_CMD_PAGEPROG);
1205         if (rc)
1206                 return rc;
1207
1208         /* Wait for R_B to signal program is complete  */
1209         rc = ndf_build_wait_busy(tn);
1210         if (rc)
1211                 return rc;
1212
1213         rc = ndf_build_post_cmd(tn, t2);
1214         if (rc)
1215                 return rc;
1216
1217         /* Wait for the DMA to complete */
1218         rc = ndf_wait(tn);
1219         if (rc)
1220                 return rc;
1221
1222         /* Data transfer is done but NDF is not, it is waiting for R/B# */
1223         return ndf_wait_idle(tn);
1224 }
1225
1226 static void octeontx_nand_cmdfunc(struct mtd_info *mtd, unsigned int command,
1227                                   int column, int page_addr)
1228 {
1229         struct nand_chip *nand = mtd_to_nand(mtd);
1230         struct octeontx_nand_chip *octeontx_nand = to_otx_nand(nand);
1231         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1232         int rc;
1233
1234         tn->selected_chip = octeontx_nand->cs;
1235         if (tn->selected_chip < 0 || tn->selected_chip >= NAND_MAX_CHIPS) {
1236                 dev_err(tn->dev, "invalid chip select\n");
1237                 return;
1238         }
1239
1240         tn->use_status = false;
1241
1242         pr_debug("%s(%p, 0x%x, 0x%x, 0x%x) cs: %d\n", __func__, mtd, command,
1243                  column, page_addr, tn->selected_chip);
1244         switch (command) {
1245         case NAND_CMD_READID:
1246                 tn->buf.data_index = 0;
1247                 octeontx_nand->oob_only = false;
1248                 rc = ndf_read(tn, command, 1, column, 0, 0, 8);
1249                 if (rc < 0)
1250                         dev_err(tn->dev, "READID failed with %d\n", rc);
1251                 else
1252                         tn->buf.data_len = rc;
1253                 break;
1254
1255         case NAND_CMD_READOOB:
1256                 octeontx_nand->oob_only = true;
1257                 tn->buf.data_index = 0;
1258                 tn->buf.data_len = 0;
1259                 rc = ndf_page_read(tn, page_addr, column, mtd->oobsize);
1260                 if (rc < mtd->oobsize)
1261                         dev_err(tn->dev, "READOOB failed with %d\n",
1262                                 tn->buf.data_len);
1263                 else
1264                         tn->buf.data_len = rc;
1265                 break;
1266
1267         case NAND_CMD_READ0:
1268                 octeontx_nand->oob_only = false;
1269                 tn->buf.data_index = 0;
1270                 tn->buf.data_len = 0;
1271                 rc = ndf_page_read(tn, page_addr, column,
1272                                    mtd->writesize + mtd->oobsize);
1273
1274                 if (rc < mtd->writesize + mtd->oobsize)
1275                         dev_err(tn->dev, "READ0 failed with %d\n", rc);
1276                 else
1277                         tn->buf.data_len = rc;
1278                 break;
1279
1280         case NAND_CMD_STATUS:
1281                 /* used in oob/not states */
1282                 tn->use_status = true;
1283                 rc = ndf_read(tn, command, 0, 0, 0, 0, 8);
1284                 if (rc < 0)
1285                         dev_err(tn->dev, "STATUS failed with %d\n", rc);
1286                 break;
1287
1288         case NAND_CMD_RESET:
1289                 /* used in oob/not states */
1290                 rc = octeontx_nand_reset(tn);
1291                 if (rc < 0)
1292                         dev_err(tn->dev, "RESET failed with %d\n", rc);
1293                 break;
1294
1295         case NAND_CMD_PARAM:
1296                 octeontx_nand->oob_only = false;
1297                 tn->buf.data_index = 0;
1298                 rc = ndf_read(tn, command, 1, 0, 0, 0,
1299                               min(tn->buf.dmabuflen, 3 * 512));
1300                 if (rc < 0)
1301                         dev_err(tn->dev, "PARAM failed with %d\n", rc);
1302                 else
1303                         tn->buf.data_len = rc;
1304                 break;
1305
1306         case NAND_CMD_RNDOUT:
1307                 tn->buf.data_index = column;
1308                 break;
1309
1310         case NAND_CMD_ERASE1:
1311                 if (ndf_block_erase(tn, page_addr))
1312                         dev_err(tn->dev, "ERASE1 failed\n");
1313                 break;
1314
1315         case NAND_CMD_ERASE2:
1316                 /* We do all erase processing in the first command, so ignore
1317                  * this one.
1318                  */
1319                 break;
1320
1321         case NAND_CMD_SEQIN:
1322                 octeontx_nand->oob_only = (column >= mtd->writesize);
1323                 tn->buf.data_index = column;
1324                 tn->buf.data_len = column;
1325
1326                 octeontx_nand->selected_page = page_addr;
1327                 break;
1328
1329         case NAND_CMD_PAGEPROG:
1330                 rc = ndf_page_write(tn, octeontx_nand->selected_page);
1331                 if (rc)
1332                         dev_err(tn->dev, "PAGEPROG failed with %d\n", rc);
1333                 break;
1334
1335         case NAND_CMD_SET_FEATURES:
1336                 octeontx_nand->oob_only = false;
1337                 /* assume tn->buf.data_len == 4 of data has been set there */
1338                 rc = octeontx_nand_set_features(mtd, nand,
1339                                                 page_addr, tn->buf.dmabuf);
1340                 if (rc)
1341                         dev_err(tn->dev, "SET_FEATURES failed with %d\n", rc);
1342                 break;
1343
1344         case NAND_CMD_GET_FEATURES:
1345                 octeontx_nand->oob_only = false;
1346                 rc = octeontx_nand_get_features(mtd, nand,
1347                                                 page_addr, tn->buf.dmabuf);
1348                 if (!rc) {
1349                         tn->buf.data_index = 0;
1350                         tn->buf.data_len = 4;
1351                 } else {
1352                         dev_err(tn->dev, "GET_FEATURES failed with %d\n", rc);
1353                 }
1354                 break;
1355
1356         default:
1357                 WARN_ON_ONCE(1);
1358                 dev_err(tn->dev, "unhandled nand cmd: %x\n", command);
1359         }
1360 }
1361
1362 static int octeontx_nand_waitfunc(struct mtd_info *mtd, struct nand_chip *chip)
1363 {
1364         struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
1365         int ret;
1366
1367         ret = ndf_wait_idle(tn);
1368         return (ret < 0) ? -EIO : 0;
1369 }
1370
1371 /* check compatibility with ONFI timing mode#N, and optionally apply */
1372 /* TODO: Implement chipnr support? */
1373 static int octeontx_nand_setup_dat_intf(struct mtd_info *mtd, int chipnr,
1374                                         const struct nand_data_interface *conf)
1375 {
1376         static const bool check_only;
1377         struct nand_chip *nand = mtd_to_nand(mtd);
1378         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1379         static u64 t_wc_n[MAX_ONFI_MODE + 2]; /* cache a mode signature */
1380         int mode; /* deduced mode number, for reporting and restricting */
1381         int rc;
1382
1383         /*
1384          * Cache timing modes for reporting, and reducing needless change.
1385          *
1386          * Challenge: caller does not pass ONFI mode#, but reporting the mode
1387          * and restricting to a maximum, or a list, are useful for diagnosing
1388          * new hardware.  So use tWC_min, distinct and monotonic across modes,
1389          * to discover the requested/accepted mode number
1390          */
1391         for (mode = MAX_ONFI_MODE; mode >= 0 && !t_wc_n[0]; mode--) {
1392                 const struct nand_sdr_timings *t;
1393
1394                 t = onfi_async_timing_mode_to_sdr_timings(mode);
1395                 if (!t)
1396                         continue;
1397                 t_wc_n[mode] = t->tWC_min;
1398         }
1399
1400         if (!conf) {
1401                 rc = -EINVAL;
1402         } else if (check_only) {
1403                 rc = 0;
1404         } else if (nand->data_interface &&
1405                         chip->iface_set && chip->iface_mode == mode) {
1406                 /*
1407                  * Cases:
1408                  * - called from nand_reset, which clears DDR timing
1409                  *   mode back to SDR.  BUT if we're already in SDR,
1410                  *   timing mode persists over resets.
1411                  *   While mtd/nand layer only supports SDR,
1412                  *   this is always safe. And this driver only supports SDR.
1413                  *
1414                  * - called from post-power-event nand_reset (maybe
1415                  *   NFC+flash power down, or system hibernate.
1416                  *   Address this when CONFIG_PM support added
1417                  */
1418                 rc = 0;
1419         } else {
1420                 rc = octeontx_nfc_chip_set_timings(chip, &conf->timings.sdr);
1421                 if (!rc) {
1422                         chip->iface_mode = mode;
1423                         chip->iface_set = true;
1424                 }
1425         }
1426         return rc;
1427 }
1428
1429 static void octeontx_bch_reset(void)
1430 {
1431 }
1432
1433 /*
1434  * Given a page, calculate the ECC code
1435  *
1436  * chip:        Pointer to NAND chip data structure
1437  * buf:         Buffer to calculate ECC on
1438  * code:        Buffer to hold ECC data
1439  *
1440  * Return 0 on success or -1 on failure
1441  */
1442 static int octeontx_nand_bch_calculate_ecc_internal(struct mtd_info *mtd,
1443                                                     dma_addr_t ihandle,
1444                                                     u8 *code)
1445 {
1446         struct nand_chip *nand = mtd_to_nand(mtd);
1447         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1448         int rc;
1449         int i;
1450         static u8 *ecc_buffer;
1451         static int ecc_size;
1452         static unsigned long ecc_handle;
1453         union bch_resp *r = tn->bch_resp;
1454
1455         if (!ecc_buffer || ecc_size < nand->ecc.size) {
1456                 ecc_size = nand->ecc.size;
1457                 ecc_buffer = dma_alloc_coherent(ecc_size,
1458                                                 (unsigned long *)&ecc_handle);
1459         }
1460
1461         memset(ecc_buffer, 0, nand->ecc.bytes);
1462
1463         r->u16 = 0;
1464         __iowmb(); /* flush done=0 before making request */
1465
1466         rc = octeontx_bch_encode(bch_vf, ihandle, nand->ecc.size,
1467                                  nand->ecc.strength,
1468                                  (dma_addr_t)ecc_handle, tn->bch_rhandle);
1469
1470         if (!rc) {
1471                 octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
1472         } else {
1473                 dev_err(tn->dev, "octeontx_bch_encode failed\n");
1474                 return -1;
1475         }
1476
1477         if (!r->s.done || r->s.uncorrectable) {
1478                 dev_err(tn->dev,
1479                         "%s timeout, done:%d uncorr:%d corr:%d erased:%d\n",
1480                         __func__, r->s.done, r->s.uncorrectable,
1481                         r->s.num_errors, r->s.erased);
1482                 octeontx_bch_reset();
1483                 return -1;
1484         }
1485
1486         memcpy(code, ecc_buffer, nand->ecc.bytes);
1487
1488         for (i = 0; i < nand->ecc.bytes; i++)
1489                 code[i] ^= tn->eccmask[i];
1490
1491         return tn->bch_resp->s.num_errors;
1492 }
1493
1494 /*
1495  * Given a page, calculate the ECC code
1496  *
1497  * mtd:        MTD block structure
1498  * dat:        raw data (unused)
1499  * ecc_code:   buffer for ECC
1500  */
1501 static int octeontx_nand_bch_calculate(struct mtd_info *mtd,
1502                                        const u8 *dat, u8 *ecc_code)
1503 {
1504         struct nand_chip *nand = mtd_to_nand(mtd);
1505         dma_addr_t handle = dma_map_single((u8 *)dat,
1506                                            nand->ecc.size, DMA_TO_DEVICE);
1507         int ret;
1508
1509         ret = octeontx_nand_bch_calculate_ecc_internal(mtd, handle,
1510                                                        (void *)ecc_code);
1511
1512         return ret;
1513 }
1514
1515 /*
1516  * Detect and correct multi-bit ECC for a page
1517  *
1518  * mtd:        MTD block structure
1519  * dat:        raw data read from the chip
1520  * read_ecc:   ECC from the chip (unused)
1521  * isnull:     unused
1522  *
1523  * Returns number of bits corrected or -1 if unrecoverable
1524  */
1525 static int octeontx_nand_bch_correct(struct mtd_info *mtd, u_char *dat,
1526                                      u_char *read_ecc, u_char *isnull)
1527 {
1528         struct nand_chip *nand = mtd_to_nand(mtd);
1529         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1530         int i = nand->ecc.size + nand->ecc.bytes;
1531         static u8 *data_buffer;
1532         static dma_addr_t ihandle;
1533         static int buffer_size;
1534         dma_addr_t ohandle;
1535         union bch_resp *r = tn->bch_resp;
1536         int rc;
1537
1538         if (i > buffer_size) {
1539                 if (buffer_size)
1540                         free(data_buffer);
1541                 data_buffer = dma_alloc_coherent(i,
1542                                                  (unsigned long *)&ihandle);
1543                 if (!data_buffer) {
1544                         dev_err(tn->dev,
1545                                 "%s: Could not allocate %d bytes for buffer\n",
1546                                 __func__, i);
1547                         goto error;
1548                 }
1549                 buffer_size = i;
1550         }
1551
1552         memcpy(data_buffer, dat, nand->ecc.size);
1553         memcpy(data_buffer + nand->ecc.size, read_ecc, nand->ecc.bytes);
1554
1555         for (i = 0; i < nand->ecc.bytes; i++)
1556                 data_buffer[nand->ecc.size + i] ^= tn->eccmask[i];
1557
1558         r->u16 = 0;
1559         __iowmb(); /* flush done=0 before making request */
1560
1561         ohandle = dma_map_single(dat, nand->ecc.size, DMA_FROM_DEVICE);
1562         rc = octeontx_bch_decode(bch_vf, ihandle, nand->ecc.size,
1563                                  nand->ecc.strength, ohandle, tn->bch_rhandle);
1564
1565         if (!rc)
1566                 octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
1567
1568         if (rc) {
1569                 dev_err(tn->dev, "octeontx_bch_decode failed\n");
1570                 goto error;
1571         }
1572
1573         if (!r->s.done) {
1574                 dev_err(tn->dev, "Error: BCH engine timeout\n");
1575                 octeontx_bch_reset();
1576                 goto error;
1577         }
1578
1579         if (r->s.erased) {
1580                 debug("Info: BCH block is erased\n");
1581                 return 0;
1582         }
1583
1584         if (r->s.uncorrectable) {
1585                 debug("Cannot correct NAND block, response: 0x%x\n",
1586                       r->u16);
1587                 goto error;
1588         }
1589
1590         return r->s.num_errors;
1591
1592 error:
1593         debug("Error performing bch correction\n");
1594         return -1;
1595 }
1596
1597 void octeontx_nand_bch_hwctl(struct mtd_info *mtd, int mode)
1598 {
1599         /* Do nothing. */
1600 }
1601
1602 static int octeontx_nand_hw_bch_read_page(struct mtd_info *mtd,
1603                                           struct nand_chip *chip, u8 *buf,
1604                                           int oob_required, int page)
1605 {
1606         struct nand_chip *nand = mtd_to_nand(mtd);
1607         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1608         int i, eccsize = chip->ecc.size, ret;
1609         int eccbytes = chip->ecc.bytes;
1610         int eccsteps = chip->ecc.steps;
1611         u8 *p;
1612         u8 *ecc_code = chip->buffers->ecccode;
1613         unsigned int max_bitflips = 0;
1614
1615         /* chip->read_buf() insists on sequential order, we do OOB first */
1616         memcpy(chip->oob_poi, tn->buf.dmabuf + mtd->writesize, mtd->oobsize);
1617
1618         /* Use private buffer as input for ECC correction */
1619         p = tn->buf.dmabuf;
1620
1621         ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
1622                                          chip->ecc.total);
1623         if (ret)
1624                 return ret;
1625
1626         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1627                 int stat;
1628
1629                 debug("Correcting block offset %lx, ecc offset %x\n",
1630                       p - buf, i);
1631                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL);
1632
1633                 if (stat < 0) {
1634                         mtd->ecc_stats.failed++;
1635                         debug("Cannot correct NAND page %d\n", page);
1636                 } else {
1637                         mtd->ecc_stats.corrected += stat;
1638                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1639                 }
1640         }
1641
1642         /* Copy corrected data to caller's buffer now */
1643         memcpy(buf, tn->buf.dmabuf, mtd->writesize);
1644
1645         return max_bitflips;
1646 }
1647
1648 static int octeontx_nand_hw_bch_write_page(struct mtd_info *mtd,
1649                                            struct nand_chip *chip,
1650                                            const u8 *buf, int oob_required,
1651                                            int page)
1652 {
1653         struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
1654         int i, eccsize = chip->ecc.size, ret;
1655         int eccbytes = chip->ecc.bytes;
1656         int eccsteps = chip->ecc.steps;
1657         const u8 *p;
1658         u8 *ecc_calc = chip->buffers->ecccalc;
1659
1660         debug("%s(buf?%p, oob%d p%x)\n",
1661               __func__, buf, oob_required, page);
1662         for (i = 0; i < chip->ecc.total; i++)
1663                 ecc_calc[i] = 0xFF;
1664
1665         /* Copy the page data from caller's buffers to private buffer */
1666         chip->write_buf(mtd, buf, mtd->writesize);
1667         /* Use private date as source for ECC calculation */
1668         p = tn->buf.dmabuf;
1669
1670         /* Hardware ECC calculation */
1671         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1672                 int ret;
1673
1674                 ret = chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1675
1676                 if (ret < 0)
1677                         debug("calculate(mtd, p?%p, &ecc_calc[%d]?%p) returned %d\n",
1678                               p, i, &ecc_calc[i], ret);
1679
1680                 debug("block offset %lx, ecc offset %x\n", p - buf, i);
1681         }
1682
1683         ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1684                                          chip->ecc.total);
1685         if (ret)
1686                 return ret;
1687
1688         /* Store resulting OOB into private buffer, will be sent to HW */
1689         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1690
1691         return 0;
1692 }
1693
1694 /**
1695  * nand_write_page_raw - [INTERN] raw page write function
1696  * @mtd: mtd info structure
1697  * @chip: nand chip info structure
1698  * @buf: data buffer
1699  * @oob_required: must write chip->oob_poi to OOB
1700  * @page: page number to write
1701  *
1702  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1703  */
1704 static int octeontx_nand_write_page_raw(struct mtd_info *mtd,
1705                                         struct nand_chip *chip,
1706                                         const u8 *buf, int oob_required,
1707                                         int page)
1708 {
1709         chip->write_buf(mtd, buf, mtd->writesize);
1710         if (oob_required)
1711                 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1712
1713         return 0;
1714 }
1715
1716 /**
1717  * octeontx_nand_write_oob_std - [REPLACEABLE] the most common OOB data write
1718  *                             function
1719  * @mtd: mtd info structure
1720  * @chip: nand chip info structure
1721  * @page: page number to write
1722  */
1723 static int octeontx_nand_write_oob_std(struct mtd_info *mtd,
1724                                        struct nand_chip *chip,
1725                                        int page)
1726 {
1727         int status = 0;
1728         const u8 *buf = chip->oob_poi;
1729         int length = mtd->oobsize;
1730
1731         chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
1732         chip->write_buf(mtd, buf, length);
1733         /* Send command to program the OOB data */
1734         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1735
1736         status = chip->waitfunc(mtd, chip);
1737
1738         return status & NAND_STATUS_FAIL ? -EIO : 0;
1739 }
1740
1741 /**
1742  * octeontx_nand_read_page_raw - [INTERN] read raw page data without ecc
1743  * @mtd: mtd info structure
1744  * @chip: nand chip info structure
1745  * @buf: buffer to store read data
1746  * @oob_required: caller requires OOB data read to chip->oob_poi
1747  * @page: page number to read
1748  *
1749  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1750  */
1751 static int octeontx_nand_read_page_raw(struct mtd_info *mtd,
1752                                        struct nand_chip *chip,
1753                                        u8 *buf, int oob_required, int page)
1754 {
1755         chip->read_buf(mtd, buf, mtd->writesize);
1756         if (oob_required)
1757                 chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1758         return 0;
1759 }
1760
1761 static int octeontx_nand_read_oob_std(struct mtd_info *mtd,
1762                                       struct nand_chip *chip,
1763                                       int page)
1764
1765 {
1766         chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
1767         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1768         return 0;
1769 }
1770
1771 static int octeontx_nand_calc_bch_ecc_strength(struct nand_chip *nand)
1772 {
1773         struct mtd_info *mtd = nand_to_mtd(nand);
1774         struct nand_ecc_ctrl *ecc = &nand->ecc;
1775         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1776         int nsteps = mtd->writesize / ecc->size;
1777         int oobchunk = mtd->oobsize / nsteps;
1778
1779         /* ecc->strength determines ecc_level and OOB's ecc_bytes. */
1780         const u8 strengths[]  = {4, 8, 16, 24, 32, 40, 48, 56, 60, 64};
1781         /* first set the desired ecc_level to match strengths[] */
1782         int index = ARRAY_SIZE(strengths) - 1;
1783         int need;
1784
1785         while (index > 0 && !(ecc->options & NAND_ECC_MAXIMIZE) &&
1786                strengths[index - 1] >= ecc->strength)
1787                 index--;
1788
1789         do {
1790                 need = DIV_ROUND_UP(15 * strengths[index], 8);
1791                 if (need <= oobchunk - 2)
1792                         break;
1793         } while (index > 0);
1794
1795         debug("%s: steps ds: %d, strength ds: %d\n", __func__,
1796               nand->ecc_step_ds, nand->ecc_strength_ds);
1797         ecc->strength = strengths[index];
1798         ecc->bytes = need;
1799         debug("%s: strength: %d, bytes: %d\n", __func__, ecc->strength,
1800               ecc->bytes);
1801
1802         if (!tn->eccmask)
1803                 tn->eccmask = devm_kzalloc(tn->dev, ecc->bytes, GFP_KERNEL);
1804         if (!tn->eccmask)
1805                 return -ENOMEM;
1806
1807         return 0;
1808 }
1809
1810 /* sample the BCH signature of an erased (all 0xff) page,
1811  * to XOR into all page traffic, so erased pages have no ECC errors
1812  */
1813 static int octeontx_bch_save_empty_eccmask(struct nand_chip *nand)
1814 {
1815         struct mtd_info *mtd = nand_to_mtd(nand);
1816         struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
1817         unsigned int eccsize = nand->ecc.size;
1818         unsigned int eccbytes = nand->ecc.bytes;
1819         u8 erased_ecc[eccbytes];
1820         unsigned long erased_handle;
1821         unsigned char *erased_page = dma_alloc_coherent(eccsize,
1822                                                         &erased_handle);
1823         int i;
1824         int rc = 0;
1825
1826         if (!erased_page)
1827                 return -ENOMEM;
1828
1829         memset(erased_page, 0xff, eccsize);
1830         memset(erased_ecc, 0, eccbytes);
1831
1832         rc = octeontx_nand_bch_calculate_ecc_internal(mtd,
1833                                                       (dma_addr_t)erased_handle,
1834                                                       erased_ecc);
1835
1836         free(erased_page);
1837
1838         for (i = 0; i < eccbytes; i++)
1839                 tn->eccmask[i] = erased_ecc[i] ^ 0xff;
1840
1841         return rc;
1842 }
1843
1844 static void octeontx_nfc_chip_sizing(struct nand_chip *nand)
1845 {
1846         struct octeontx_nand_chip *chip = to_otx_nand(nand);
1847         struct mtd_info *mtd = nand_to_mtd(nand);
1848         struct nand_ecc_ctrl *ecc = &nand->ecc;
1849
1850         chip->row_bytes = nand->onfi_params.addr_cycles & 0xf;
1851         chip->col_bytes = nand->onfi_params.addr_cycles >> 4;
1852         debug("%s(%p) row bytes: %d, col bytes: %d, ecc mode: %d\n",
1853               __func__, nand, chip->row_bytes, chip->col_bytes, ecc->mode);
1854
1855         /*
1856          * HW_BCH using OcteonTX BCH engine, or SOFT_BCH laid out in
1857          * HW_BCH-compatible fashion, depending on devtree advice
1858          * and kernel config.
1859          * BCH/NFC hardware capable of subpage ops, not implemented.
1860          */
1861         mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
1862         nand->options |= NAND_NO_SUBPAGE_WRITE;
1863         debug("%s: start steps: %d, size: %d, bytes: %d\n",
1864               __func__, ecc->steps, ecc->size, ecc->bytes);
1865         debug("%s: step ds: %d, strength ds: %d\n", __func__,
1866               nand->ecc_step_ds, nand->ecc_strength_ds);
1867
1868         if (ecc->mode != NAND_ECC_NONE) {
1869                 int nsteps = ecc->steps ? ecc->steps : 1;
1870
1871                 if (ecc->size && ecc->size != mtd->writesize)
1872                         nsteps = mtd->writesize / ecc->size;
1873                 else if (mtd->writesize > def_ecc_size &&
1874                          !(mtd->writesize & (def_ecc_size - 1)))
1875                         nsteps = mtd->writesize / def_ecc_size;
1876                 ecc->steps = nsteps;
1877                 ecc->size = mtd->writesize / nsteps;
1878                 ecc->bytes = mtd->oobsize / nsteps;
1879
1880                 if (nand->ecc_strength_ds)
1881                         ecc->strength = nand->ecc_strength_ds;
1882                 if (nand->ecc_step_ds)
1883                         ecc->size = nand->ecc_step_ds;
1884                 /*
1885                  * no subpage ops, but set subpage-shift to match ecc->steps
1886                  * so mtd_nandbiterrs tests appropriate boundaries
1887                  */
1888                 if (!mtd->subpage_sft && !(ecc->steps & (ecc->steps - 1)))
1889                         mtd->subpage_sft = fls(ecc->steps) - 1;
1890
1891                 if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
1892                         debug("%s: ecc mode: %d\n", __func__, ecc->mode);
1893                         if (ecc->mode != NAND_ECC_SOFT &&
1894                             !octeontx_nand_calc_bch_ecc_strength(nand)) {
1895                                 struct octeontx_nfc *tn =
1896                                         to_otx_nfc(nand->controller);
1897
1898                                 debug("Using hardware BCH engine support\n");
1899                                 ecc->mode = NAND_ECC_HW_SYNDROME;
1900                                 ecc->read_page = octeontx_nand_hw_bch_read_page;
1901                                 ecc->write_page =
1902                                         octeontx_nand_hw_bch_write_page;
1903                                 ecc->read_page_raw =
1904                                         octeontx_nand_read_page_raw;
1905                                 ecc->write_page_raw =
1906                                         octeontx_nand_write_page_raw;
1907                                 ecc->read_oob = octeontx_nand_read_oob_std;
1908                                 ecc->write_oob = octeontx_nand_write_oob_std;
1909
1910                                 ecc->calculate = octeontx_nand_bch_calculate;
1911                                 ecc->correct = octeontx_nand_bch_correct;
1912                                 ecc->hwctl = octeontx_nand_bch_hwctl;
1913
1914                                 debug("NAND chip %d using hw_bch\n",
1915                                       tn->selected_chip);
1916                                 debug(" %d bytes ECC per %d byte block\n",
1917                                       ecc->bytes, ecc->size);
1918                                 debug(" for %d bits of correction per block.",
1919                                       ecc->strength);
1920                                 octeontx_nand_calc_ecc_layout(nand);
1921                                 octeontx_bch_save_empty_eccmask(nand);
1922                         }
1923                 }
1924         }
1925 }
1926
1927 static int octeontx_nfc_chip_init(struct octeontx_nfc *tn, struct udevice *dev,
1928                                   ofnode node)
1929 {
1930         struct octeontx_nand_chip *chip;
1931         struct nand_chip *nand;
1932         struct mtd_info *mtd;
1933         int ret;
1934
1935         chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
1936         if (!chip)
1937                 return -ENOMEM;
1938
1939         debug("%s: Getting chip select\n", __func__);
1940         ret = ofnode_read_s32(node, "reg", &chip->cs);
1941         if (ret) {
1942                 dev_err(dev, "could not retrieve reg property: %d\n", ret);
1943                 return ret;
1944         }
1945
1946         if (chip->cs >= NAND_MAX_CHIPS) {
1947                 dev_err(dev, "invalid reg value: %u (max CS = 7)\n", chip->cs);
1948                 return -EINVAL;
1949         }
1950         debug("%s: chip select: %d\n", __func__, chip->cs);
1951         nand = &chip->nand;
1952         nand->controller = &tn->controller;
1953         if (!tn->controller.active)
1954                 tn->controller.active = nand;
1955
1956         debug("%s: Setting flash node\n", __func__);
1957         nand_set_flash_node(nand, node);
1958
1959         nand->options = 0;
1960         nand->select_chip = octeontx_nand_select_chip;
1961         nand->cmdfunc = octeontx_nand_cmdfunc;
1962         nand->waitfunc = octeontx_nand_waitfunc;
1963         nand->read_byte = octeontx_nand_read_byte;
1964         nand->read_buf = octeontx_nand_read_buf;
1965         nand->write_buf = octeontx_nand_write_buf;
1966         nand->onfi_set_features = octeontx_nand_set_features;
1967         nand->onfi_get_features = octeontx_nand_get_features;
1968         nand->setup_data_interface = octeontx_nand_setup_dat_intf;
1969
1970         mtd = nand_to_mtd(nand);
1971         debug("%s: mtd: %p\n", __func__, mtd);
1972         mtd->dev->parent = dev;
1973
1974         debug("%s: NDF_MISC: 0x%llx\n", __func__,
1975               readq(tn->base + NDF_MISC));
1976
1977         /* TODO: support more then 1 chip */
1978         debug("%s: Scanning identification\n", __func__);
1979         ret = nand_scan_ident(mtd, 1, NULL);
1980         if (ret)
1981                 return ret;
1982
1983         debug("%s: Sizing chip\n", __func__);
1984         octeontx_nfc_chip_sizing(nand);
1985
1986         debug("%s: Scanning tail\n", __func__);
1987         ret = nand_scan_tail(mtd);
1988         if (ret) {
1989                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
1990                 return ret;
1991         }
1992
1993         debug("%s: Registering mtd\n", __func__);
1994         ret = nand_register(0, mtd);
1995
1996         debug("%s: Adding tail\n", __func__);
1997         list_add_tail(&chip->node, &tn->chips);
1998         return 0;
1999 }
2000
2001 static int octeontx_nfc_chips_init(struct octeontx_nfc *tn)
2002 {
2003         struct udevice *dev = tn->dev;
2004         ofnode node = dev_ofnode(dev);
2005         ofnode nand_node;
2006         int nr_chips = of_get_child_count(node);
2007         int ret;
2008
2009         debug("%s: node: %s\n", __func__, ofnode_get_name(node));
2010         debug("%s: %d chips\n", __func__, nr_chips);
2011         if (nr_chips > NAND_MAX_CHIPS) {
2012                 dev_err(dev, "too many NAND chips: %d\n", nr_chips);
2013                 return -EINVAL;
2014         }
2015
2016         if (!nr_chips) {
2017                 debug("no DT NAND chips found\n");
2018                 return -ENODEV;
2019         }
2020
2021         pr_info("%s: scanning %d chips DTs\n", __func__, nr_chips);
2022
2023         ofnode_for_each_subnode(nand_node, node) {
2024                 debug("%s: Calling octeontx_nfc_chip_init(%p, %s, %ld)\n",
2025                       __func__, tn, dev->name, nand_node.of_offset);
2026                 ret = octeontx_nfc_chip_init(tn, dev, nand_node);
2027                 if (ret)
2028                         return ret;
2029         }
2030         return 0;
2031 }
2032
2033 /* Reset NFC and initialize registers. */
2034 static int octeontx_nfc_init(struct octeontx_nfc *tn)
2035 {
2036         const struct nand_sdr_timings *timings;
2037         u64 ndf_misc;
2038         int rc;
2039
2040         /* Initialize values and reset the fifo */
2041         ndf_misc = readq(tn->base + NDF_MISC);
2042
2043         ndf_misc &= ~NDF_MISC_EX_DIS;
2044         ndf_misc |= (NDF_MISC_BT_DIS | NDF_MISC_RST_FF);
2045         writeq(ndf_misc, tn->base + NDF_MISC);
2046         debug("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
2047
2048         /* Bring the fifo out of reset */
2049         ndf_misc &= ~(NDF_MISC_RST_FF);
2050
2051         /* Maximum of co-processor cycles for glitch filtering */
2052         ndf_misc |= FIELD_PREP(NDF_MISC_WAIT_CNT, 0x3f);
2053
2054         writeq(ndf_misc, tn->base + NDF_MISC);
2055
2056         /* Set timing parameters to onfi mode 0 for probing */
2057         timings = onfi_async_timing_mode_to_sdr_timings(0);
2058         if (IS_ERR(timings))
2059                 return PTR_ERR(timings);
2060         rc = set_default_timings(tn, timings);
2061         if (rc)
2062                 return rc;
2063
2064         return 0;
2065 }
2066
2067 static int octeontx_pci_nand_probe(struct udevice *dev)
2068 {
2069         struct octeontx_nfc *tn = dev_get_priv(dev);
2070         int ret;
2071         static bool probe_done;
2072
2073         debug("%s(%s) tn: %p\n", __func__, dev->name, tn);
2074         if (probe_done)
2075                 return 0;
2076
2077         if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
2078                 bch_vf = octeontx_bch_getv();
2079                 if (!bch_vf) {
2080                         struct octeontx_probe_device *probe_dev;
2081
2082                         debug("%s: bch not yet initialized\n", __func__);
2083                         probe_dev = calloc(sizeof(*probe_dev), 1);
2084                         if (!probe_dev) {
2085                                 printf("%s: Out of memory\n", __func__);
2086                                 return -ENOMEM;
2087                         }
2088                         probe_dev->dev = dev;
2089                         INIT_LIST_HEAD(&probe_dev->list);
2090                         list_add_tail(&probe_dev->list,
2091                                       &octeontx_pci_nand_deferred_devices);
2092                         debug("%s: Defering probe until after BCH initialization\n",
2093                               __func__);
2094                         return 0;
2095                 }
2096         }
2097
2098         tn->dev = dev;
2099         INIT_LIST_HEAD(&tn->chips);
2100
2101         tn->base = dm_pci_map_bar(dev, PCI_BASE_ADDRESS_0, 0, 0, PCI_REGION_TYPE, PCI_REGION_MEM);
2102         if (!tn->base) {
2103                 ret = -EINVAL;
2104                 goto release;
2105         }
2106         debug("%s: bar at %p\n", __func__, tn->base);
2107         tn->buf.dmabuflen = NAND_MAX_PAGESIZE + NAND_MAX_OOBSIZE;
2108         tn->buf.dmabuf = dma_alloc_coherent(tn->buf.dmabuflen,
2109                                             (unsigned long *)&tn->buf.dmaaddr);
2110         if (!tn->buf.dmabuf) {
2111                 ret = -ENOMEM;
2112                 debug("%s: Could not allocate DMA buffer\n", __func__);
2113                 goto unclk;
2114         }
2115
2116         /* one hw-bch response, for one outstanding transaction */
2117         tn->bch_resp = dma_alloc_coherent(sizeof(*tn->bch_resp),
2118                                           (unsigned long *)&tn->bch_rhandle);
2119
2120         tn->stat = dma_alloc_coherent(8, (unsigned long *)&tn->stat_addr);
2121         if (!tn->stat || !tn->bch_resp) {
2122                 debug("%s: Could not allocate bch status or response\n",
2123                       __func__);
2124                 ret = -ENOMEM;
2125                 goto unclk;
2126         }
2127
2128         debug("%s: Calling octeontx_nfc_init()\n", __func__);
2129         octeontx_nfc_init(tn);
2130         debug("%s: Initializing chips\n", __func__);
2131         ret = octeontx_nfc_chips_init(tn);
2132         debug("%s: init chips ret: %d\n", __func__, ret);
2133         if (ret) {
2134                 if (ret != -ENODEV)
2135                         dev_err(dev, "failed to init nand chips\n");
2136                 goto unclk;
2137         }
2138         dev_info(dev, "probed\n");
2139         return 0;
2140
2141 unclk:
2142 release:
2143         return ret;
2144 }
2145
2146 int octeontx_pci_nand_disable(struct udevice *dev)
2147 {
2148         struct octeontx_nfc *tn = dev_get_priv(dev);
2149         u64 dma_cfg;
2150         u64 ndf_misc;
2151
2152         debug("%s: Disabling NAND device %s\n", __func__, dev->name);
2153         dma_cfg = readq(tn->base + NDF_DMA_CFG);
2154         dma_cfg &= ~NDF_DMA_CFG_EN;
2155         dma_cfg |= NDF_DMA_CFG_CLR;
2156         writeq(dma_cfg, tn->base + NDF_DMA_CFG);
2157
2158         /* Disable execution and put FIFO in reset mode */
2159         ndf_misc = readq(tn->base + NDF_MISC);
2160         ndf_misc |= NDF_MISC_EX_DIS | NDF_MISC_RST_FF;
2161         writeq(ndf_misc, tn->base + NDF_MISC);
2162         ndf_misc &= ~NDF_MISC_RST_FF;
2163         writeq(ndf_misc, tn->base + NDF_MISC);
2164 #ifdef DEBUG
2165         printf("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
2166 #endif
2167         /* Clear any interrupts and enable bits */
2168         writeq(~0ull, tn->base + NDF_INT_ENA_W1C);
2169         writeq(~0ull, tn->base + NDF_INT);
2170         debug("%s: NDF_ST_REG: 0x%llx\n", __func__,
2171               readq(tn->base + NDF_ST_REG));
2172         return 0;
2173 }
2174
2175 /**
2176  * Since it's possible (and even likely) that the NAND device will be probed
2177  * before the BCH device has been probed, we may need to defer the probing.
2178  *
2179  * In this case, the initial probe returns success but the actual probing
2180  * is deferred until the BCH VF has been probed.
2181  *
2182  * Return:      0 for success, otherwise error
2183  */
2184 int octeontx_pci_nand_deferred_probe(void)
2185 {
2186         int rc = 0;
2187         struct octeontx_probe_device *pdev;
2188
2189         debug("%s: Performing deferred probing\n", __func__);
2190         list_for_each_entry(pdev, &octeontx_pci_nand_deferred_devices, list) {
2191                 debug("%s: Probing %s\n", __func__, pdev->dev->name);
2192                 dev_get_flags(pdev->dev) &= ~DM_FLAG_ACTIVATED;
2193                 rc = device_probe(pdev->dev);
2194                 if (rc && rc != -ENODEV) {
2195                         printf("%s: Error %d with deferred probe of %s\n",
2196                                __func__, rc, pdev->dev->name);
2197                         break;
2198                 }
2199         }
2200         return rc;
2201 }
2202
2203 static const struct pci_device_id octeontx_nfc_pci_id_table[] = {
2204         { PCI_VDEVICE(CAVIUM, 0xA04F) },
2205         {}
2206 };
2207
2208 static int octeontx_nand_of_to_plat(struct udevice *dev)
2209 {
2210         return 0;
2211 }
2212
2213 static const struct udevice_id octeontx_nand_ids[] = {
2214         { .compatible = "cavium,cn8130-nand" },
2215         { },
2216 };
2217
2218 U_BOOT_DRIVER(octeontx_pci_nand) = {
2219         .name   = OCTEONTX_NAND_DRIVER_NAME,
2220         .id     = UCLASS_MTD,
2221         .of_match = of_match_ptr(octeontx_nand_ids),
2222         .of_to_plat = octeontx_nand_of_to_plat,
2223         .probe = octeontx_pci_nand_probe,
2224         .priv_auto      = sizeof(struct octeontx_nfc),
2225         .remove = octeontx_pci_nand_disable,
2226         .flags = DM_FLAG_OS_PREPARE,
2227 };
2228
2229 U_BOOT_PCI_DEVICE(octeontx_pci_nand, octeontx_nfc_pci_id_table);
2230
2231 void board_nand_init(void)
2232 {
2233         struct udevice *dev;
2234         int ret;
2235
2236         if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
2237                 ret = uclass_get_device_by_driver(UCLASS_MISC,
2238                                                   DM_DRIVER_GET(octeontx_pci_bchpf),
2239                                                   &dev);
2240                 if (ret && ret != -ENODEV) {
2241                         pr_err("Failed to initialize OcteonTX BCH PF controller. (error %d)\n",
2242                                ret);
2243                 }
2244                 ret = uclass_get_device_by_driver(UCLASS_MISC,
2245                                                   DM_DRIVER_GET(octeontx_pci_bchvf),
2246                                                   &dev);
2247                 if (ret && ret != -ENODEV) {
2248                         pr_err("Failed to initialize OcteonTX BCH VF controller. (error %d)\n",
2249                                ret);
2250                 }
2251         }
2252
2253         ret = uclass_get_device_by_driver(UCLASS_MTD,
2254                                           DM_DRIVER_GET(octeontx_pci_nand),
2255                                           &dev);
2256         if (ret && ret != -ENODEV)
2257                 pr_err("Failed to initialize OcteonTX NAND controller. (error %d)\n",
2258                        ret);
2259 }