drivers/mtd/nand/raw/meson_nand.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
   2 /*
   3  * Amlogic Meson Nand Flash Controller Driver
   4  *
   5  * Copyright (c) 2018 Amlogic, inc.
   6  * Author: Liang Yang <liang.yang@amlogic.com>
   7  */
   8
   9 #include <linux/platform_device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/clk.h>
  13 #include <linux/clk-provider.h>
  14 #include <linux/mtd/rawnand.h>
  15 #include <linux/mtd/mtd.h>
  16 #include <linux/mfd/syscon.h>
  17 #include <linux/regmap.h>
  18 #include <linux/slab.h>
  19 #include <linux/module.h>
  20 #include <linux/iopoll.h>
  21 #include <linux/of.h>
  22 #include <linux/of_device.h>
  23 #include <linux/sched/task_stack.h>
  24
  25 #define NFC_REG_CMD             0x00
  26 #define NFC_CMD_IDLE            (0xc << 14)
  27 #define NFC_CMD_CLE             (0x5 << 14)
  28 #define NFC_CMD_ALE             (0x6 << 14)
  29 #define NFC_CMD_ADL             ((0 << 16) | (3 << 20))
  30 #define NFC_CMD_ADH             ((1 << 16) | (3 << 20))
  31 #define NFC_CMD_AIL             ((2 << 16) | (3 << 20))
  32 #define NFC_CMD_AIH             ((3 << 16) | (3 << 20))
  33 #define NFC_CMD_SEED            ((8 << 16) | (3 << 20))
  34 #define NFC_CMD_M2N             ((0 << 17) | (2 << 20))
  35 #define NFC_CMD_N2M             ((1 << 17) | (2 << 20))
  36 #define NFC_CMD_RB              BIT(20)
  37 #define NFC_CMD_SCRAMBLER_ENABLE        BIT(19)
  38 #define NFC_CMD_SCRAMBLER_DISABLE       0
  39 #define NFC_CMD_SHORTMODE_DISABLE       0
  40 #define NFC_CMD_RB_INT          BIT(14)
  41 #define NFC_CMD_RB_INT_NO_PIN   ((0xb << 10) | BIT(18) | BIT(16))
  42
  43 #define NFC_CMD_GET_SIZE(x)     (((x) >> 22) & GENMASK(4, 0))
  44
  45 #define NFC_REG_CFG             0x04
  46 #define NFC_REG_DADR            0x08
  47 #define NFC_REG_IADR            0x0c
  48 #define NFC_REG_BUF             0x10
  49 #define NFC_REG_INFO            0x14
  50 #define NFC_REG_DC              0x18
  51 #define NFC_REG_ADR             0x1c
  52 #define NFC_REG_DL              0x20
  53 #define NFC_REG_DH              0x24
  54 #define NFC_REG_CADR            0x28
  55 #define NFC_REG_SADR            0x2c
  56 #define NFC_REG_PINS            0x30
  57 #define NFC_REG_VER             0x38
  58
  59 #define NFC_RB_IRQ_EN           BIT(21)
  60
  61 #define CLK_DIV_SHIFT           0
  62 #define CLK_DIV_WIDTH           6
  63
  64 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)       \
  65         (                                                               \
  66                 (cmd_dir)                       |                       \
  67                 ((ran) << 19)                   |                       \
  68                 ((bch) << 14)                   |                       \
  69                 ((short_mode) << 13)            |                       \
  70                 (((page_size) & 0x7f) << 6)     |                       \
  71                 ((pages) & 0x3f)                                        \
  72         )
  73
  74 #define GENCMDDADDRL(adl, addr)         ((adl) | ((addr) & 0xffff))
  75 #define GENCMDDADDRH(adh, addr)         ((adh) | (((addr) >> 16) & 0xffff))
  76 #define GENCMDIADDRL(ail, addr)         ((ail) | ((addr) & 0xffff))
  77 #define GENCMDIADDRH(aih, addr)         ((aih) | (((addr) >> 16) & 0xffff))
  78
  79 #define DMA_DIR(dir)            ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
  80 #define DMA_ADDR_ALIGN          8
  81
  82 #define ECC_CHECK_RETURN_FF     (-1)
  83
  84 #define NAND_CE0                (0xe << 10)
  85 #define NAND_CE1                (0xd << 10)
  86
  87 #define DMA_BUSY_TIMEOUT        0x100000
  88 #define CMD_FIFO_EMPTY_TIMEOUT  1000
  89
  90 #define MAX_CE_NUM              2
  91
  92 /* eMMC clock register, misc control */
  93 #define CLK_SELECT_NAND         BIT(31)
  94
  95 #define NFC_CLK_CYCLE           6
  96
  97 /* nand flash controller delay 3 ns */
  98 #define NFC_DEFAULT_DELAY       3000
  99
 100 #define ROW_ADDER(page, index)  (((page) >> (8 * (index))) & 0xff)
 101 #define MAX_CYCLE_ADDRS         5
 102 #define DIRREAD                 1
 103 #define DIRWRITE                0
 104
 105 #define ECC_PARITY_BCH8_512B    14
 106 #define ECC_COMPLETE            BIT(31)
 107 #define ECC_ERR_CNT(x)          (((x) >> 24) & GENMASK(5, 0))
 108 #define ECC_ZERO_CNT(x)         (((x) >> 16) & GENMASK(5, 0))
 109 #define ECC_UNCORRECTABLE       0x3f
 110
 111 #define PER_INFO_BYTE           8
 112
 113 #define NFC_CMD_RAW_LEN GENMASK(13, 0)
 114
 115 #define NFC_COLUMN_ADDR_0       0
 116 #define NFC_COLUMN_ADDR_1       0
 117
 118 struct meson_nfc_nand_chip {
 119         struct list_head node;
 120         struct nand_chip nand;
 121         unsigned long clk_rate;
 122         unsigned long level1_divider;
 123         u32 bus_timing;
 124         u32 twb;
 125         u32 tadl;
 126         u32 tbers_max;
 127
 128         u32 bch_mode;
 129         u8 *data_buf;
 130         __le64 *info_buf;
 131         u32 nsels;
 132         u8 sels[];
 133 };
 134
 135 struct meson_nand_ecc {
 136         u32 bch;
 137         u32 strength;
 138 };
 139
 140 struct meson_nfc_data {
 141         const struct nand_ecc_caps *ecc_caps;
 142 };
 143
 144 struct meson_nfc_param {
 145         u32 chip_select;
 146         u32 rb_select;
 147 };
 148
 149 struct nand_rw_cmd {
 150         u32 cmd0;
 151         u32 addrs[MAX_CYCLE_ADDRS];
 152         u32 cmd1;
 153 };
 154
 155 struct nand_timing {
 156         u32 twb;
 157         u32 tadl;
 158         u32 tbers_max;
 159 };
 160
 161 struct meson_nfc {
 162         struct nand_controller controller;
 163         struct clk *core_clk;
 164         struct clk *device_clk;
 165         struct clk *nand_clk;
 166         struct clk_divider nand_divider;
 167
 168         unsigned long clk_rate;
 169         u32 bus_timing;
 170
 171         struct device *dev;
 172         void __iomem *reg_base;
 173         void __iomem *reg_clk;
 174         struct completion completion;
 175         struct list_head chips;
 176         const struct meson_nfc_data *data;
 177         struct meson_nfc_param param;
 178         struct nand_timing timing;
 179         union {
 180                 int cmd[32];
 181                 struct nand_rw_cmd rw;
 182         } cmdfifo;
 183
 184         dma_addr_t daddr;
 185         dma_addr_t iaddr;
 186         u32 info_bytes;
 187
 188         unsigned long assigned_cs;
 189         bool no_rb_pin;
 190 };
 191
 192 enum {
 193         NFC_ECC_BCH8_1K         = 2,
 194         NFC_ECC_BCH24_1K,
 195         NFC_ECC_BCH30_1K,
 196         NFC_ECC_BCH40_1K,
 197         NFC_ECC_BCH50_1K,
 198         NFC_ECC_BCH60_1K,
 199 };
 200
 201 #define MESON_ECC_DATA(b, s)    { .bch = (b),   .strength = (s)}
 202
 203 static struct meson_nand_ecc meson_ecc[] = {
 204         MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8),
 205         MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24),
 206         MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30),
 207         MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40),
 208         MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50),
 209         MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60),
 210 };
 211
 212 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
 213 {
 214         int ecc_bytes;
 215
 216         if (step_size == 512 && strength == 8)
 217                 return ECC_PARITY_BCH8_512B;
 218
 219         ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
 220         ecc_bytes = ALIGN(ecc_bytes, 2);
 221
 222         return ecc_bytes;
 223 }
 224
 225 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
 226                      meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
 227 NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps,
 228                      meson_nand_calc_ecc_bytes, 1024, 8);
 229
 230 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
 231 {
 232         return container_of(nand, struct meson_nfc_nand_chip, nand);
 233 }
 234
 235 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
 236 {
 237         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 238         struct meson_nfc *nfc = nand_get_controller_data(nand);
 239         int ret, value;
 240
 241         if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
 242                 return;
 243
 244         nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
 245         nfc->param.rb_select = nfc->param.chip_select;
 246         nfc->timing.twb = meson_chip->twb;
 247         nfc->timing.tadl = meson_chip->tadl;
 248         nfc->timing.tbers_max = meson_chip->tbers_max;
 249
 250         if (nfc->clk_rate != meson_chip->clk_rate) {
 251                 ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
 252                 if (ret) {
 253                         dev_err(nfc->dev, "failed to set clock rate\n");
 254                         return;
 255                 }
 256                 nfc->clk_rate = meson_chip->clk_rate;
 257         }
 258         if (nfc->bus_timing != meson_chip->bus_timing) {
 259                 value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
 260                 writel(value, nfc->reg_base + NFC_REG_CFG);
 261                 writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
 262                 nfc->bus_timing =  meson_chip->bus_timing;
 263         }
 264 }
 265
 266 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
 267 {
 268         writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
 269                nfc->reg_base + NFC_REG_CMD);
 270 }
 271
 272 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
 273 {
 274         writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
 275                nfc->reg_base + NFC_REG_CMD);
 276 }
 277
 278 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
 279                                  int scrambler)
 280 {
 281         struct mtd_info *mtd = nand_to_mtd(nand);
 282         struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
 283         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 284         u32 bch = meson_chip->bch_mode, cmd;
 285         int len = mtd->writesize, pagesize, pages;
 286
 287         pagesize = nand->ecc.size;
 288
 289         if (raw) {
 290                 len = mtd->writesize + mtd->oobsize;
 291                 cmd = len | scrambler | DMA_DIR(dir);
 292                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 293                 return;
 294         }
 295
 296         pages = len / nand->ecc.size;
 297
 298         cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
 299                        NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
 300
 301         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 302 }
 303
 304 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
 305 {
 306         /*
 307          * Insert two commands to make sure all valid commands are finished.
 308          *
 309          * The Nand flash controller is designed as two stages pipleline -
 310          *  a) fetch and b) excute.
 311          * There might be cases when the driver see command queue is empty,
 312          * but the Nand flash controller still has two commands buffered,
 313          * one is fetched into NFC request queue (ready to run), and another
 314          * is actively executing. So pushing 2 "IDLE" commands guarantees that
 315          * the pipeline is emptied.
 316          */
 317         meson_nfc_cmd_idle(nfc, 0);
 318         meson_nfc_cmd_idle(nfc, 0);
 319 }
 320
 321 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
 322                                      unsigned int timeout_ms)
 323 {
 324         u32 cmd_size = 0;
 325         int ret;
 326
 327         /* wait cmd fifo is empty */
 328         ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
 329                                          !NFC_CMD_GET_SIZE(cmd_size),
 330                                          10, timeout_ms * 1000);
 331         if (ret)
 332                 dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
 333
 334         return ret;
 335 }
 336
 337 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
 338 {
 339         meson_nfc_drain_cmd(nfc);
 340
 341         return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
 342 }
 343
 344 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
 345 {
 346         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 347         int len;
 348
 349         len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
 350
 351         return meson_chip->data_buf + len;
 352 }
 353
 354 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
 355 {
 356         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 357         int len, temp;
 358
 359         temp = nand->ecc.size + nand->ecc.bytes;
 360         len = (temp + 2) * i;
 361
 362         return meson_chip->data_buf + len;
 363 }
 364
 365 static void meson_nfc_get_data_oob(struct nand_chip *nand,
 366                                    u8 *buf, u8 *oobbuf)
 367 {
 368         int i, oob_len = 0;
 369         u8 *dsrc, *osrc;
 370
 371         oob_len = nand->ecc.bytes + 2;
 372         for (i = 0; i < nand->ecc.steps; i++) {
 373                 if (buf) {
 374                         dsrc = meson_nfc_data_ptr(nand, i);
 375                         memcpy(buf, dsrc, nand->ecc.size);
 376                         buf += nand->ecc.size;
 377                 }
 378                 osrc = meson_nfc_oob_ptr(nand, i);
 379                 memcpy(oobbuf, osrc, oob_len);
 380                 oobbuf += oob_len;
 381         }
 382 }
 383
 384 static void meson_nfc_set_data_oob(struct nand_chip *nand,
 385                                    const u8 *buf, u8 *oobbuf)
 386 {
 387         int i, oob_len = 0;
 388         u8 *dsrc, *osrc;
 389
 390         oob_len = nand->ecc.bytes + 2;
 391         for (i = 0; i < nand->ecc.steps; i++) {
 392                 if (buf) {
 393                         dsrc = meson_nfc_data_ptr(nand, i);
 394                         memcpy(dsrc, buf, nand->ecc.size);
 395                         buf += nand->ecc.size;
 396                 }
 397                 osrc = meson_nfc_oob_ptr(nand, i);
 398                 memcpy(osrc, oobbuf, oob_len);
 399                 oobbuf += oob_len;
 400         }
 401 }
 402
 403 static int meson_nfc_wait_no_rb_pin(struct meson_nfc *nfc, int timeout_ms,
 404                                     bool need_cmd_read0)
 405 {
 406         u32 cmd, cfg;
 407
 408         meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 409         meson_nfc_drain_cmd(nfc);
 410         meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 411
 412         cfg = readl(nfc->reg_base + NFC_REG_CFG);
 413         cfg |= NFC_RB_IRQ_EN;
 414         writel(cfg, nfc->reg_base + NFC_REG_CFG);
 415
 416         reinit_completion(&nfc->completion);
 417         cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_STATUS;
 418         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 419
 420         /* use the max erase time as the maximum clock for waiting R/B */
 421         cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
 422         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 423
 424         if (!wait_for_completion_timeout(&nfc->completion,
 425                                          msecs_to_jiffies(timeout_ms)))
 426                 return -ETIMEDOUT;
 427
 428         if (need_cmd_read0) {
 429                 cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_READ0;
 430                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 431                 meson_nfc_drain_cmd(nfc);
 432                 meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 433         }
 434
 435         return 0;
 436 }
 437
 438 static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
 439 {
 440         u32 cmd, cfg;
 441         int ret = 0;
 442
 443         meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 444         meson_nfc_drain_cmd(nfc);
 445         meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 446
 447         cfg = readl(nfc->reg_base + NFC_REG_CFG);
 448         cfg |= NFC_RB_IRQ_EN;
 449         writel(cfg, nfc->reg_base + NFC_REG_CFG);
 450
 451         reinit_completion(&nfc->completion);
 452
 453         /* use the max erase time as the maximum clock for waiting R/B */
 454         cmd = NFC_CMD_RB | NFC_CMD_RB_INT
 455                 | nfc->param.chip_select | nfc->timing.tbers_max;
 456         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 457
 458         ret = wait_for_completion_timeout(&nfc->completion,
 459                                           msecs_to_jiffies(timeout_ms));
 460         if (ret == 0)
 461                 ret = -1;
 462
 463         return ret;
 464 }
 465
 466 static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms,
 467                               bool need_cmd_read0)
 468 {
 469         if (nfc->no_rb_pin) {
 470                 /* This mode is used when there is no wired R/B pin.
 471                  * It works like 'nand_soft_waitrdy()', but instead of
 472                  * polling NAND_CMD_STATUS bit in the software loop,
 473                  * it will wait for interrupt - controllers checks IO
 474                  * bus and when it detects NAND_CMD_STATUS on it, it
 475                  * raises interrupt. After interrupt, NAND_CMD_READ0 is
 476                  * sent as terminator of the ready waiting procedure if
 477                  * needed (for all cases except page programming - this
 478                  * is reason of 'need_cmd_read0' flag).
 479                  */
 480                 return meson_nfc_wait_no_rb_pin(nfc, timeout_ms,
 481                                                 need_cmd_read0);
 482         } else {
 483                 return meson_nfc_wait_rb_pin(nfc, timeout_ms);
 484         }
 485 }
 486
 487 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
 488 {
 489         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 490         __le64 *info;
 491         int i, count;
 492
 493         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 494                 info = &meson_chip->info_buf[i];
 495                 *info |= oob_buf[count];
 496                 *info |= oob_buf[count + 1] << 8;
 497         }
 498 }
 499
 500 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
 501 {
 502         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 503         __le64 *info;
 504         int i, count;
 505
 506         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 507                 info = &meson_chip->info_buf[i];
 508                 oob_buf[count] = *info;
 509                 oob_buf[count + 1] = *info >> 8;
 510         }
 511 }
 512
 513 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
 514                                  u64 *correct_bitmap)
 515 {
 516         struct mtd_info *mtd = nand_to_mtd(nand);
 517         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 518         __le64 *info;
 519         int ret = 0, i;
 520
 521         for (i = 0; i < nand->ecc.steps; i++) {
 522                 info = &meson_chip->info_buf[i];
 523                 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
 524                         mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
 525                         *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
 526                         *correct_bitmap |= BIT_ULL(i);
 527                         continue;
 528                 }
 529                 if ((nand->options & NAND_NEED_SCRAMBLING) &&
 530                     ECC_ZERO_CNT(*info) < nand->ecc.strength) {
 531                         mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
 532                         *bitflips = max_t(u32, *bitflips,
 533                                           ECC_ZERO_CNT(*info));
 534                         ret = ECC_CHECK_RETURN_FF;
 535                 } else {
 536                         ret = -EBADMSG;
 537                 }
 538         }
 539         return ret;
 540 }
 541
 542 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
 543                                       int datalen, void *infobuf, int infolen,
 544                                       enum dma_data_direction dir)
 545 {
 546         struct meson_nfc *nfc = nand_get_controller_data(nand);
 547         u32 cmd;
 548         int ret = 0;
 549
 550         nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
 551         ret = dma_mapping_error(nfc->dev, nfc->daddr);
 552         if (ret) {
 553                 dev_err(nfc->dev, "DMA mapping error\n");
 554                 return ret;
 555         }
 556         cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
 557         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 558
 559         cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
 560         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 561
 562         if (infobuf) {
 563                 nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
 564                 ret = dma_mapping_error(nfc->dev, nfc->iaddr);
 565                 if (ret) {
 566                         dev_err(nfc->dev, "DMA mapping error\n");
 567                         dma_unmap_single(nfc->dev,
 568                                          nfc->daddr, datalen, dir);
 569                         return ret;
 570                 }
 571                 nfc->info_bytes = infolen;
 572                 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
 573                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 574
 575                 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
 576                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 577         }
 578
 579         return ret;
 580 }
 581
 582 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
 583                                          int datalen, int infolen,
 584                                          enum dma_data_direction dir)
 585 {
 586         struct meson_nfc *nfc = nand_get_controller_data(nand);
 587
 588         dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
 589         if (infolen) {
 590                 dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
 591                 nfc->info_bytes = 0;
 592         }
 593 }
 594
 595 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
 596 {
 597         struct meson_nfc *nfc = nand_get_controller_data(nand);
 598         int ret = 0;
 599         u32 cmd;
 600         u8 *info;
 601
 602         info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
 603         if (!info)
 604                 return -ENOMEM;
 605
 606         ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
 607                                          PER_INFO_BYTE, DMA_FROM_DEVICE);
 608         if (ret)
 609                 goto out;
 610
 611         cmd = NFC_CMD_N2M | len;
 612         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 613
 614         meson_nfc_drain_cmd(nfc);
 615         meson_nfc_wait_cmd_finish(nfc, 1000);
 616         meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
 617
 618 out:
 619         kfree(info);
 620
 621         return ret;
 622 }
 623
 624 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
 625 {
 626         struct meson_nfc *nfc = nand_get_controller_data(nand);
 627         int ret = 0;
 628         u32 cmd;
 629
 630         ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
 631                                          0, DMA_TO_DEVICE);
 632         if (ret)
 633                 return ret;
 634
 635         cmd = NFC_CMD_M2N | len;
 636         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 637
 638         meson_nfc_drain_cmd(nfc);
 639         meson_nfc_wait_cmd_finish(nfc, 1000);
 640         meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
 641
 642         return ret;
 643 }
 644
 645 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
 646                                                 int page, bool in)
 647 {
 648         const struct nand_sdr_timings *sdr =
 649                 nand_get_sdr_timings(nand_get_interface_config(nand));
 650         struct mtd_info *mtd = nand_to_mtd(nand);
 651         struct meson_nfc *nfc = nand_get_controller_data(nand);
 652         u32 *addrs = nfc->cmdfifo.rw.addrs;
 653         u32 cs = nfc->param.chip_select;
 654         u32 cmd0, cmd_num, row_start;
 655         int i;
 656
 657         cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
 658
 659         cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
 660         nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
 661
 662         addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
 663         if (mtd->writesize <= 512) {
 664                 cmd_num--;
 665                 row_start = 1;
 666         } else {
 667                 addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
 668                 row_start = 2;
 669         }
 670
 671         addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
 672         addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
 673
 674         if (nand->options & NAND_ROW_ADDR_3)
 675                 addrs[row_start + 2] =
 676                         cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
 677         else
 678                 cmd_num--;
 679
 680         /* subtract cmd1 */
 681         cmd_num--;
 682
 683         for (i = 0; i < cmd_num; i++)
 684                 writel_relaxed(nfc->cmdfifo.cmd[i],
 685                                nfc->reg_base + NFC_REG_CMD);
 686
 687         if (in) {
 688                 nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
 689                 writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
 690                 meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max), true);
 691         } else {
 692                 meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
 693         }
 694
 695         return 0;
 696 }
 697
 698 static int meson_nfc_write_page_sub(struct nand_chip *nand,
 699                                     int page, int raw)
 700 {
 701         const struct nand_sdr_timings *sdr =
 702                 nand_get_sdr_timings(nand_get_interface_config(nand));
 703         struct mtd_info *mtd = nand_to_mtd(nand);
 704         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 705         struct meson_nfc *nfc = nand_get_controller_data(nand);
 706         int data_len, info_len;
 707         u32 cmd;
 708         int ret;
 709
 710         meson_nfc_select_chip(nand, nand->cur_cs);
 711
 712         data_len =  mtd->writesize + mtd->oobsize;
 713         info_len = nand->ecc.steps * PER_INFO_BYTE;
 714
 715         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
 716         if (ret)
 717                 return ret;
 718
 719         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 720                                          data_len, meson_chip->info_buf,
 721                                          info_len, DMA_TO_DEVICE);
 722         if (ret)
 723                 return ret;
 724
 725         if (nand->options & NAND_NEED_SCRAMBLING) {
 726                 meson_nfc_cmd_seed(nfc, page);
 727                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 728                                      NFC_CMD_SCRAMBLER_ENABLE);
 729         } else {
 730                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 731                                      NFC_CMD_SCRAMBLER_DISABLE);
 732         }
 733
 734         cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
 735         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 736         meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max), false);
 737
 738         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
 739
 740         return ret;
 741 }
 742
 743 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
 744                                     int oob_required, int page)
 745 {
 746         u8 *oob_buf = nand->oob_poi;
 747
 748         meson_nfc_set_data_oob(nand, buf, oob_buf);
 749
 750         return meson_nfc_write_page_sub(nand, page, 1);
 751 }
 752
 753 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
 754                                       const u8 *buf, int oob_required, int page)
 755 {
 756         struct mtd_info *mtd = nand_to_mtd(nand);
 757         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 758         u8 *oob_buf = nand->oob_poi;
 759
 760         memcpy(meson_chip->data_buf, buf, mtd->writesize);
 761         memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
 762         meson_nfc_set_user_byte(nand, oob_buf);
 763
 764         return meson_nfc_write_page_sub(nand, page, 0);
 765 }
 766
 767 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
 768                                             struct nand_chip *nand, int raw)
 769 {
 770         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 771         __le64 *info;
 772         u32 neccpages;
 773         int ret;
 774
 775         neccpages = raw ? 1 : nand->ecc.steps;
 776         info = &meson_chip->info_buf[neccpages - 1];
 777         do {
 778                 usleep_range(10, 15);
 779                 /* info is updated by nfc dma engine*/
 780                 smp_rmb();
 781                 dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
 782                                         DMA_FROM_DEVICE);
 783                 ret = *info & ECC_COMPLETE;
 784         } while (!ret);
 785 }
 786
 787 static int meson_nfc_read_page_sub(struct nand_chip *nand,
 788                                    int page, int raw)
 789 {
 790         struct mtd_info *mtd = nand_to_mtd(nand);
 791         struct meson_nfc *nfc = nand_get_controller_data(nand);
 792         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 793         int data_len, info_len;
 794         int ret;
 795
 796         meson_nfc_select_chip(nand, nand->cur_cs);
 797
 798         data_len =  mtd->writesize + mtd->oobsize;
 799         info_len = nand->ecc.steps * PER_INFO_BYTE;
 800
 801         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
 802         if (ret)
 803                 return ret;
 804
 805         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 806                                          data_len, meson_chip->info_buf,
 807                                          info_len, DMA_FROM_DEVICE);
 808         if (ret)
 809                 return ret;
 810
 811         if (nand->options & NAND_NEED_SCRAMBLING) {
 812                 meson_nfc_cmd_seed(nfc, page);
 813                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 814                                      NFC_CMD_SCRAMBLER_ENABLE);
 815         } else {
 816                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 817                                      NFC_CMD_SCRAMBLER_DISABLE);
 818         }
 819
 820         ret = meson_nfc_wait_dma_finish(nfc);
 821         meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
 822
 823         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
 824
 825         return ret;
 826 }
 827
 828 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
 829                                    int oob_required, int page)
 830 {
 831         u8 *oob_buf = nand->oob_poi;
 832         int ret;
 833
 834         ret = meson_nfc_read_page_sub(nand, page, 1);
 835         if (ret)
 836                 return ret;
 837
 838         meson_nfc_get_data_oob(nand, buf, oob_buf);
 839
 840         return 0;
 841 }
 842
 843 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
 844                                      int oob_required, int page)
 845 {
 846         struct mtd_info *mtd = nand_to_mtd(nand);
 847         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 848         struct nand_ecc_ctrl *ecc = &nand->ecc;
 849         u64 correct_bitmap = 0;
 850         u32 bitflips = 0;
 851         u8 *oob_buf = nand->oob_poi;
 852         int ret, i;
 853
 854         ret = meson_nfc_read_page_sub(nand, page, 0);
 855         if (ret)
 856                 return ret;
 857
 858         meson_nfc_get_user_byte(nand, oob_buf);
 859         ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
 860         if (ret == ECC_CHECK_RETURN_FF) {
 861                 if (buf)
 862                         memset(buf, 0xff, mtd->writesize);
 863                 memset(oob_buf, 0xff, mtd->oobsize);
 864         } else if (ret < 0) {
 865                 if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
 866                         mtd->ecc_stats.failed++;
 867                         return bitflips;
 868                 }
 869                 ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
 870                 if (ret)
 871                         return ret;
 872
 873                 for (i = 0; i < nand->ecc.steps ; i++) {
 874                         u8 *data = buf + i * ecc->size;
 875                         u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
 876
 877                         if (correct_bitmap & BIT_ULL(i))
 878                                 continue;
 879                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
 880                                                           oob, ecc->bytes + 2,
 881                                                           NULL, 0,
 882                                                           ecc->strength);
 883                         if (ret < 0) {
 884                                 mtd->ecc_stats.failed++;
 885                         } else {
 886                                 mtd->ecc_stats.corrected += ret;
 887                                 bitflips =  max_t(u32, bitflips, ret);
 888                         }
 889                 }
 890         } else if (buf && buf != meson_chip->data_buf) {
 891                 memcpy(buf, meson_chip->data_buf, mtd->writesize);
 892         }
 893
 894         return bitflips;
 895 }
 896
 897 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
 898 {
 899         return meson_nfc_read_page_raw(nand, NULL, 1, page);
 900 }
 901
 902 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
 903 {
 904         return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
 905 }
 906
 907 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
 908 {
 909         if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
 910                 return false;
 911
 912         if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
 913                 return true;
 914         return false;
 915 }
 916
 917 static void *
 918 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
 919 {
 920         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
 921                 return NULL;
 922
 923         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
 924                 return instr->ctx.data.buf.in;
 925
 926         return kzalloc(instr->ctx.data.len, GFP_KERNEL);
 927 }
 928
 929 static void
 930 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
 931                                      void *buf)
 932 {
 933         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
 934             WARN_ON(!buf))
 935                 return;
 936
 937         if (buf == instr->ctx.data.buf.in)
 938                 return;
 939
 940         memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
 941         kfree(buf);
 942 }
 943
 944 static void *
 945 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
 946 {
 947         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
 948                 return NULL;
 949
 950         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
 951                 return (void *)instr->ctx.data.buf.out;
 952
 953         return kmemdup(instr->ctx.data.buf.out,
 954                        instr->ctx.data.len, GFP_KERNEL);
 955 }
 956
 957 static void
 958 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
 959                                       const void *buf)
 960 {
 961         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
 962             WARN_ON(!buf))
 963                 return;
 964
 965         if (buf != instr->ctx.data.buf.out)
 966                 kfree(buf);
 967 }
 968
 969 static int meson_nfc_check_op(struct nand_chip *chip,
 970                               const struct nand_operation *op)
 971 {
 972         int op_id;
 973
 974         for (op_id = 0; op_id < op->ninstrs; op_id++) {
 975                 const struct nand_op_instr *instr;
 976
 977                 instr = &op->instrs[op_id];
 978
 979                 switch (instr->type) {
 980                 case NAND_OP_DATA_IN_INSTR:
 981                 case NAND_OP_DATA_OUT_INSTR:
 982                         if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
 983                                 return -ENOTSUPP;
 984
 985                         break;
 986                 default:
 987                         break;
 988                 }
 989         }
 990
 991         return 0;
 992 }
 993
 994 static int meson_nfc_exec_op(struct nand_chip *nand,
 995                              const struct nand_operation *op, bool check_only)
 996 {
 997         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 998         struct meson_nfc *nfc = nand_get_controller_data(nand);
 999         const struct nand_op_instr *instr = NULL;
1000         void *buf;
1001         u32 op_id, delay_idle, cmd;
1002         int err;
1003         int i;
1004
1005         err = meson_nfc_check_op(nand, op);
1006         if (err)
1007                 return err;
1008
1009         if (check_only)
1010                 return 0;
1011
1012         meson_nfc_select_chip(nand, op->cs);
1013         for (op_id = 0; op_id < op->ninstrs; op_id++) {
1014                 instr = &op->instrs[op_id];
1015                 delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1016                                           meson_chip->level1_divider *
1017                                           NFC_CLK_CYCLE);
1018                 switch (instr->type) {
1019                 case NAND_OP_CMD_INSTR:
1020                         cmd = nfc->param.chip_select | NFC_CMD_CLE;
1021                         cmd |= instr->ctx.cmd.opcode & 0xff;
1022                         writel(cmd, nfc->reg_base + NFC_REG_CMD);
1023                         meson_nfc_cmd_idle(nfc, delay_idle);
1024                         break;
1025
1026                 case NAND_OP_ADDR_INSTR:
1027                         for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1028                                 cmd = nfc->param.chip_select | NFC_CMD_ALE;
1029                                 cmd |= instr->ctx.addr.addrs[i] & 0xff;
1030                                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
1031                         }
1032                         meson_nfc_cmd_idle(nfc, delay_idle);
1033                         break;
1034
1035                 case NAND_OP_DATA_IN_INSTR:
1036                         buf = meson_nand_op_get_dma_safe_input_buf(instr);
1037                         if (!buf)
1038                                 return -ENOMEM;
1039                         meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
1040                         meson_nand_op_put_dma_safe_input_buf(instr, buf);
1041                         break;
1042
1043                 case NAND_OP_DATA_OUT_INSTR:
1044                         buf = meson_nand_op_get_dma_safe_output_buf(instr);
1045                         if (!buf)
1046                                 return -ENOMEM;
1047                         meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
1048                         meson_nand_op_put_dma_safe_output_buf(instr, buf);
1049                         break;
1050
1051                 case NAND_OP_WAITRDY_INSTR:
1052                         meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms,
1053                                            true);
1054                         if (instr->delay_ns)
1055                                 meson_nfc_cmd_idle(nfc, delay_idle);
1056                         break;
1057                 }
1058         }
1059         meson_nfc_wait_cmd_finish(nfc, 1000);
1060         return 0;
1061 }
1062
1063 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1064                                struct mtd_oob_region *oobregion)
1065 {
1066         struct nand_chip *nand = mtd_to_nand(mtd);
1067
1068         if (section >= nand->ecc.steps)
1069                 return -ERANGE;
1070
1071         oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
1072         oobregion->length = nand->ecc.bytes;
1073
1074         return 0;
1075 }
1076
1077 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1078                                 struct mtd_oob_region *oobregion)
1079 {
1080         struct nand_chip *nand = mtd_to_nand(mtd);
1081
1082         if (section >= nand->ecc.steps)
1083                 return -ERANGE;
1084
1085         oobregion->offset = section * (2 + nand->ecc.bytes);
1086         oobregion->length = 2;
1087
1088         return 0;
1089 }
1090
1091 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1092         .ecc = meson_ooblayout_ecc,
1093         .free = meson_ooblayout_free,
1094 };
1095
1096 static int meson_nfc_clk_init(struct meson_nfc *nfc)
1097 {
1098         struct clk_parent_data nfc_divider_parent_data[1] = {0};
1099         struct clk_init_data init = {0};
1100         int ret;
1101
1102         /* request core clock */
1103         nfc->core_clk = devm_clk_get(nfc->dev, "core");
1104         if (IS_ERR(nfc->core_clk)) {
1105                 dev_err(nfc->dev, "failed to get core clock\n");
1106                 return PTR_ERR(nfc->core_clk);
1107         }
1108
1109         nfc->device_clk = devm_clk_get(nfc->dev, "device");
1110         if (IS_ERR(nfc->device_clk)) {
1111                 dev_err(nfc->dev, "failed to get device clock\n");
1112                 return PTR_ERR(nfc->device_clk);
1113         }
1114
1115         init.name = devm_kasprintf(nfc->dev,
1116                                    GFP_KERNEL, "%s#div",
1117                                    dev_name(nfc->dev));
1118         init.ops = &clk_divider_ops;
1119         nfc_divider_parent_data[0].fw_name = "device";
1120         init.parent_data = nfc_divider_parent_data;
1121         init.num_parents = 1;
1122         nfc->nand_divider.reg = nfc->reg_clk;
1123         nfc->nand_divider.shift = CLK_DIV_SHIFT;
1124         nfc->nand_divider.width = CLK_DIV_WIDTH;
1125         nfc->nand_divider.hw.init = &init;
1126         nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1127                                   CLK_DIVIDER_ROUND_CLOSEST |
1128                                   CLK_DIVIDER_ALLOW_ZERO;
1129
1130         nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1131         if (IS_ERR(nfc->nand_clk))
1132                 return PTR_ERR(nfc->nand_clk);
1133
1134         /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1135         writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
1136                nfc->reg_clk);
1137
1138         ret = clk_prepare_enable(nfc->core_clk);
1139         if (ret) {
1140                 dev_err(nfc->dev, "failed to enable core clock\n");
1141                 return ret;
1142         }
1143
1144         ret = clk_prepare_enable(nfc->device_clk);
1145         if (ret) {
1146                 dev_err(nfc->dev, "failed to enable device clock\n");
1147                 goto err_device_clk;
1148         }
1149
1150         ret = clk_prepare_enable(nfc->nand_clk);
1151         if (ret) {
1152                 dev_err(nfc->dev, "pre enable NFC divider fail\n");
1153                 goto err_nand_clk;
1154         }
1155
1156         ret = clk_set_rate(nfc->nand_clk, 24000000);
1157         if (ret)
1158                 goto err_disable_clk;
1159
1160         return 0;
1161
1162 err_disable_clk:
1163         clk_disable_unprepare(nfc->nand_clk);
1164 err_nand_clk:
1165         clk_disable_unprepare(nfc->device_clk);
1166 err_device_clk:
1167         clk_disable_unprepare(nfc->core_clk);
1168         return ret;
1169 }
1170
1171 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1172 {
1173         clk_disable_unprepare(nfc->nand_clk);
1174         clk_disable_unprepare(nfc->device_clk);
1175         clk_disable_unprepare(nfc->core_clk);
1176 }
1177
1178 static void meson_nfc_free_buffer(struct nand_chip *nand)
1179 {
1180         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1181
1182         kfree(meson_chip->info_buf);
1183         kfree(meson_chip->data_buf);
1184 }
1185
1186 static int meson_chip_buffer_init(struct nand_chip *nand)
1187 {
1188         struct mtd_info *mtd = nand_to_mtd(nand);
1189         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1190         u32 page_bytes, info_bytes, nsectors;
1191
1192         nsectors = mtd->writesize / nand->ecc.size;
1193
1194         page_bytes =  mtd->writesize + mtd->oobsize;
1195         info_bytes = nsectors * PER_INFO_BYTE;
1196
1197         meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1198         if (!meson_chip->data_buf)
1199                 return -ENOMEM;
1200
1201         meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1202         if (!meson_chip->info_buf) {
1203                 kfree(meson_chip->data_buf);
1204                 return -ENOMEM;
1205         }
1206
1207         return 0;
1208 }
1209
1210 static
1211 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1212                               const struct nand_interface_config *conf)
1213 {
1214         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1215         const struct nand_sdr_timings *timings;
1216         u32 div, bt_min, bt_max, tbers_clocks;
1217
1218         timings = nand_get_sdr_timings(conf);
1219         if (IS_ERR(timings))
1220                 return -ENOTSUPP;
1221
1222         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1223                 return 0;
1224
1225         div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1226         bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1227         bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1228                   timings->tRC_min / 2) / div;
1229
1230         meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1231                                        div * NFC_CLK_CYCLE);
1232         meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1233                                         div * NFC_CLK_CYCLE);
1234         tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1235                                         div * NFC_CLK_CYCLE);
1236         meson_chip->tbers_max = ilog2(tbers_clocks);
1237         if (!is_power_of_2(tbers_clocks))
1238                 meson_chip->tbers_max++;
1239
1240         bt_min = DIV_ROUND_UP(bt_min, 1000);
1241         bt_max = DIV_ROUND_UP(bt_max, 1000);
1242
1243         if (bt_max < bt_min)
1244                 return -EINVAL;
1245
1246         meson_chip->level1_divider = div;
1247         meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1248         meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1249
1250         return 0;
1251 }
1252
1253 static int meson_nand_bch_mode(struct nand_chip *nand)
1254 {
1255         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1256         int i;
1257
1258         if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1259                 return -EINVAL;
1260
1261         for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1262                 if (meson_ecc[i].strength == nand->ecc.strength) {
1263                         meson_chip->bch_mode = meson_ecc[i].bch;
1264                         return 0;
1265                 }
1266         }
1267
1268         return -EINVAL;
1269 }
1270
1271 static void meson_nand_detach_chip(struct nand_chip *nand)
1272 {
1273         meson_nfc_free_buffer(nand);
1274 }
1275
1276 static int meson_nand_attach_chip(struct nand_chip *nand)
1277 {
1278         struct meson_nfc *nfc = nand_get_controller_data(nand);
1279         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1280         struct mtd_info *mtd = nand_to_mtd(nand);
1281         int raw_writesize;
1282         int ret;
1283
1284         if (!mtd->name) {
1285                 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1286                                            "%s:nand%d",
1287                                            dev_name(nfc->dev),
1288                                            meson_chip->sels[0]);
1289                 if (!mtd->name)
1290                         return -ENOMEM;
1291         }
1292
1293         raw_writesize = mtd->writesize + mtd->oobsize;
1294         if (raw_writesize > NFC_CMD_RAW_LEN) {
1295                 dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1296                         raw_writesize, NFC_CMD_RAW_LEN);
1297                 return -EINVAL;
1298         }
1299
1300         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1301                 nand->bbt_options |= NAND_BBT_NO_OOB;
1302
1303         nand->options |= NAND_NO_SUBPAGE_WRITE;
1304
1305         ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1306                                    mtd->oobsize - 2);
1307         if (ret) {
1308                 dev_err(nfc->dev, "failed to ECC init\n");
1309                 return -EINVAL;
1310         }
1311
1312         mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1313
1314         ret = meson_nand_bch_mode(nand);
1315         if (ret)
1316                 return -EINVAL;
1317
1318         nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1319         nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1320         nand->ecc.write_page = meson_nfc_write_page_hwecc;
1321         nand->ecc.write_oob_raw = nand_write_oob_std;
1322         nand->ecc.write_oob = nand_write_oob_std;
1323
1324         nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1325         nand->ecc.read_page = meson_nfc_read_page_hwecc;
1326         nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1327         nand->ecc.read_oob = meson_nfc_read_oob;
1328
1329         if (nand->options & NAND_BUSWIDTH_16) {
1330                 dev_err(nfc->dev, "16bits bus width not supported");
1331                 return -EINVAL;
1332         }
1333         ret = meson_chip_buffer_init(nand);
1334         if (ret)
1335                 return -ENOMEM;
1336
1337         return ret;
1338 }
1339
1340 static const struct nand_controller_ops meson_nand_controller_ops = {
1341         .attach_chip = meson_nand_attach_chip,
1342         .detach_chip = meson_nand_detach_chip,
1343         .setup_interface = meson_nfc_setup_interface,
1344         .exec_op = meson_nfc_exec_op,
1345 };
1346
1347 static int
1348 meson_nfc_nand_chip_init(struct device *dev,
1349                          struct meson_nfc *nfc, struct device_node *np)
1350 {
1351         struct meson_nfc_nand_chip *meson_chip;
1352         struct nand_chip *nand;
1353         struct mtd_info *mtd;
1354         int ret, i;
1355         u32 tmp, nsels;
1356         u32 nand_rb_val = 0;
1357
1358         nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1359         if (!nsels || nsels > MAX_CE_NUM) {
1360                 dev_err(dev, "invalid register property size\n");
1361                 return -EINVAL;
1362         }
1363
1364         meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1365                                   GFP_KERNEL);
1366         if (!meson_chip)
1367                 return -ENOMEM;
1368
1369         meson_chip->nsels = nsels;
1370
1371         for (i = 0; i < nsels; i++) {
1372                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1373                 if (ret) {
1374                         dev_err(dev, "could not retrieve register property: %d\n",
1375                                 ret);
1376                         return ret;
1377                 }
1378
1379                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1380                         dev_err(dev, "CS %d already assigned\n", tmp);
1381                         return -EINVAL;
1382                 }
1383         }
1384
1385         nand = &meson_chip->nand;
1386         nand->controller = &nfc->controller;
1387         nand->controller->ops = &meson_nand_controller_ops;
1388         nand_set_flash_node(nand, np);
1389         nand_set_controller_data(nand, nfc);
1390
1391         nand->options |= NAND_USES_DMA;
1392         mtd = nand_to_mtd(nand);
1393         mtd->owner = THIS_MODULE;
1394         mtd->dev.parent = dev;
1395
1396         ret = of_property_read_u32(np, "nand-rb", &nand_rb_val);
1397         if (ret == -EINVAL)
1398                 nfc->no_rb_pin = true;
1399         else if (ret)
1400                 return ret;
1401
1402         if (nand_rb_val)
1403                 return -EINVAL;
1404
1405         ret = nand_scan(nand, nsels);
1406         if (ret)
1407                 return ret;
1408
1409         ret = mtd_device_register(mtd, NULL, 0);
1410         if (ret) {
1411                 dev_err(dev, "failed to register MTD device: %d\n", ret);
1412                 nand_cleanup(nand);
1413                 return ret;
1414         }
1415
1416         list_add_tail(&meson_chip->node, &nfc->chips);
1417
1418         return 0;
1419 }
1420
1421 static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1422 {
1423         struct meson_nfc_nand_chip *meson_chip;
1424         struct mtd_info *mtd;
1425
1426         while (!list_empty(&nfc->chips)) {
1427                 meson_chip = list_first_entry(&nfc->chips,
1428                                               struct meson_nfc_nand_chip, node);
1429                 mtd = nand_to_mtd(&meson_chip->nand);
1430                 WARN_ON(mtd_device_unregister(mtd));
1431
1432                 nand_cleanup(&meson_chip->nand);
1433                 list_del(&meson_chip->node);
1434         }
1435 }
1436
1437 static int meson_nfc_nand_chips_init(struct device *dev,
1438                                      struct meson_nfc *nfc)
1439 {
1440         struct device_node *np = dev->of_node;
1441         struct device_node *nand_np;
1442         int ret;
1443
1444         for_each_child_of_node(np, nand_np) {
1445                 ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1446                 if (ret) {
1447                         meson_nfc_nand_chip_cleanup(nfc);
1448                         of_node_put(nand_np);
1449                         return ret;
1450                 }
1451         }
1452
1453         return 0;
1454 }
1455
1456 static irqreturn_t meson_nfc_irq(int irq, void *id)
1457 {
1458         struct meson_nfc *nfc = id;
1459         u32 cfg;
1460
1461         cfg = readl(nfc->reg_base + NFC_REG_CFG);
1462         if (!(cfg & NFC_RB_IRQ_EN))
1463                 return IRQ_NONE;
1464
1465         cfg &= ~(NFC_RB_IRQ_EN);
1466         writel(cfg, nfc->reg_base + NFC_REG_CFG);
1467
1468         complete(&nfc->completion);
1469         return IRQ_HANDLED;
1470 }
1471
1472 static const struct meson_nfc_data meson_gxl_data = {
1473         .ecc_caps = &meson_gxl_ecc_caps,
1474 };
1475
1476 static const struct meson_nfc_data meson_axg_data = {
1477         .ecc_caps = &meson_axg_ecc_caps,
1478 };
1479
1480 static const struct of_device_id meson_nfc_id_table[] = {
1481         {
1482                 .compatible = "amlogic,meson-gxl-nfc",
1483                 .data = &meson_gxl_data,
1484         }, {
1485                 .compatible = "amlogic,meson-axg-nfc",
1486                 .data = &meson_axg_data,
1487         },
1488         {}
1489 };
1490 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1491
1492 static int meson_nfc_probe(struct platform_device *pdev)
1493 {
1494         struct device *dev = &pdev->dev;
1495         struct meson_nfc *nfc;
1496         int ret, irq;
1497
1498         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1499         if (!nfc)
1500                 return -ENOMEM;
1501
1502         nfc->data = of_device_get_match_data(&pdev->dev);
1503         if (!nfc->data)
1504                 return -ENODEV;
1505
1506         nand_controller_init(&nfc->controller);
1507         INIT_LIST_HEAD(&nfc->chips);
1508         init_completion(&nfc->completion);
1509
1510         nfc->dev = dev;
1511
1512         nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1513         if (IS_ERR(nfc->reg_base))
1514                 return PTR_ERR(nfc->reg_base);
1515
1516         nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1517         if (IS_ERR(nfc->reg_clk))
1518                 return PTR_ERR(nfc->reg_clk);
1519
1520         irq = platform_get_irq(pdev, 0);
1521         if (irq < 0)
1522                 return -EINVAL;
1523
1524         ret = meson_nfc_clk_init(nfc);
1525         if (ret) {
1526                 dev_err(dev, "failed to initialize NAND clock\n");
1527                 return ret;
1528         }
1529
1530         writel(0, nfc->reg_base + NFC_REG_CFG);
1531         ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1532         if (ret) {
1533                 dev_err(dev, "failed to request NFC IRQ\n");
1534                 ret = -EINVAL;
1535                 goto err_clk;
1536         }
1537
1538         ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1539         if (ret) {
1540                 dev_err(dev, "failed to set DMA mask\n");
1541                 goto err_clk;
1542         }
1543
1544         platform_set_drvdata(pdev, nfc);
1545
1546         ret = meson_nfc_nand_chips_init(dev, nfc);
1547         if (ret) {
1548                 dev_err(dev, "failed to init NAND chips\n");
1549                 goto err_clk;
1550         }
1551
1552         return 0;
1553 err_clk:
1554         meson_nfc_disable_clk(nfc);
1555         return ret;
1556 }
1557
1558 static void meson_nfc_remove(struct platform_device *pdev)
1559 {
1560         struct meson_nfc *nfc = platform_get_drvdata(pdev);
1561
1562         meson_nfc_nand_chip_cleanup(nfc);
1563
1564         meson_nfc_disable_clk(nfc);
1565 }
1566
1567 static struct platform_driver meson_nfc_driver = {
1568         .probe  = meson_nfc_probe,
1569         .remove_new = meson_nfc_remove,
1570         .driver = {
1571                 .name  = "meson-nand",
1572                 .of_match_table = meson_nfc_id_table,
1573         },
1574 };
1575 module_platform_driver(meson_nfc_driver);
1576
1577 MODULE_LICENSE("Dual MIT/GPL");
1578 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1579 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");