88af7145a51a2cb62280f94d9f380f80c982445e
[platform/kernel/linux-rpi.git] / drivers / mtd / nand / qcom_nandc.c
1 /*
2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
3  *
4  * This software is licensed under the terms of the GNU General Public
5  * License version 2, as published by the Free Software Foundation, and
6  * may be copied, distributed, and modified under those terms.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  */
13
14 #include <linux/clk.h>
15 #include <linux/slab.h>
16 #include <linux/bitops.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmaengine.h>
19 #include <linux/module.h>
20 #include <linux/mtd/nand.h>
21 #include <linux/mtd/partitions.h>
22 #include <linux/of.h>
23 #include <linux/of_device.h>
24 #include <linux/delay.h>
25
26 /* NANDc reg offsets */
27 #define NAND_FLASH_CMD                  0x00
28 #define NAND_ADDR0                      0x04
29 #define NAND_ADDR1                      0x08
30 #define NAND_FLASH_CHIP_SELECT          0x0c
31 #define NAND_EXEC_CMD                   0x10
32 #define NAND_FLASH_STATUS               0x14
33 #define NAND_BUFFER_STATUS              0x18
34 #define NAND_DEV0_CFG0                  0x20
35 #define NAND_DEV0_CFG1                  0x24
36 #define NAND_DEV0_ECC_CFG               0x28
37 #define NAND_DEV1_ECC_CFG               0x2c
38 #define NAND_DEV1_CFG0                  0x30
39 #define NAND_DEV1_CFG1                  0x34
40 #define NAND_READ_ID                    0x40
41 #define NAND_READ_STATUS                0x44
42 #define NAND_DEV_CMD0                   0xa0
43 #define NAND_DEV_CMD1                   0xa4
44 #define NAND_DEV_CMD2                   0xa8
45 #define NAND_DEV_CMD_VLD                0xac
46 #define SFLASHC_BURST_CFG               0xe0
47 #define NAND_ERASED_CW_DETECT_CFG       0xe8
48 #define NAND_ERASED_CW_DETECT_STATUS    0xec
49 #define NAND_EBI2_ECC_BUF_CFG           0xf0
50 #define FLASH_BUF_ACC                   0x100
51
52 #define NAND_CTRL                       0xf00
53 #define NAND_VERSION                    0xf08
54 #define NAND_READ_LOCATION_0            0xf20
55 #define NAND_READ_LOCATION_1            0xf24
56
57 /* dummy register offsets, used by write_reg_dma */
58 #define NAND_DEV_CMD1_RESTORE           0xdead
59 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
60
61 /* NAND_FLASH_CMD bits */
62 #define PAGE_ACC                        BIT(4)
63 #define LAST_PAGE                       BIT(5)
64
65 /* NAND_FLASH_CHIP_SELECT bits */
66 #define NAND_DEV_SEL                    0
67 #define DM_EN                           BIT(2)
68
69 /* NAND_FLASH_STATUS bits */
70 #define FS_OP_ERR                       BIT(4)
71 #define FS_READY_BSY_N                  BIT(5)
72 #define FS_MPU_ERR                      BIT(8)
73 #define FS_DEVICE_STS_ERR               BIT(16)
74 #define FS_DEVICE_WP                    BIT(23)
75
76 /* NAND_BUFFER_STATUS bits */
77 #define BS_UNCORRECTABLE_BIT            BIT(8)
78 #define BS_CORRECTABLE_ERR_MSK          0x1f
79
80 /* NAND_DEVn_CFG0 bits */
81 #define DISABLE_STATUS_AFTER_WRITE      4
82 #define CW_PER_PAGE                     6
83 #define UD_SIZE_BYTES                   9
84 #define ECC_PARITY_SIZE_BYTES_RS        19
85 #define SPARE_SIZE_BYTES                23
86 #define NUM_ADDR_CYCLES                 27
87 #define STATUS_BFR_READ                 30
88 #define SET_RD_MODE_AFTER_STATUS        31
89
90 /* NAND_DEVn_CFG0 bits */
91 #define DEV0_CFG1_ECC_DISABLE           0
92 #define WIDE_FLASH                      1
93 #define NAND_RECOVERY_CYCLES            2
94 #define CS_ACTIVE_BSY                   5
95 #define BAD_BLOCK_BYTE_NUM              6
96 #define BAD_BLOCK_IN_SPARE_AREA         16
97 #define WR_RD_BSY_GAP                   17
98 #define ENABLE_BCH_ECC                  27
99
100 /* NAND_DEV0_ECC_CFG bits */
101 #define ECC_CFG_ECC_DISABLE             0
102 #define ECC_SW_RESET                    1
103 #define ECC_MODE                        4
104 #define ECC_PARITY_SIZE_BYTES_BCH       8
105 #define ECC_NUM_DATA_BYTES              16
106 #define ECC_FORCE_CLK_OPEN              30
107
108 /* NAND_DEV_CMD1 bits */
109 #define READ_ADDR                       0
110
111 /* NAND_DEV_CMD_VLD bits */
112 #define READ_START_VLD                  0
113
114 /* NAND_EBI2_ECC_BUF_CFG bits */
115 #define NUM_STEPS                       0
116
117 /* NAND_ERASED_CW_DETECT_CFG bits */
118 #define ERASED_CW_ECC_MASK              1
119 #define AUTO_DETECT_RES                 0
120 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
121 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
122 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
123 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
124 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
125
126 /* NAND_ERASED_CW_DETECT_STATUS bits */
127 #define PAGE_ALL_ERASED                 BIT(7)
128 #define CODEWORD_ALL_ERASED             BIT(6)
129 #define PAGE_ERASED                     BIT(5)
130 #define CODEWORD_ERASED                 BIT(4)
131 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
132 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
133
134 /* Version Mask */
135 #define NAND_VERSION_MAJOR_MASK         0xf0000000
136 #define NAND_VERSION_MAJOR_SHIFT        28
137 #define NAND_VERSION_MINOR_MASK         0x0fff0000
138 #define NAND_VERSION_MINOR_SHIFT        16
139
140 /* NAND OP_CMDs */
141 #define PAGE_READ                       0x2
142 #define PAGE_READ_WITH_ECC              0x3
143 #define PAGE_READ_WITH_ECC_SPARE        0x4
144 #define PROGRAM_PAGE                    0x6
145 #define PAGE_PROGRAM_WITH_ECC           0x7
146 #define PROGRAM_PAGE_SPARE              0x9
147 #define BLOCK_ERASE                     0xa
148 #define FETCH_ID                        0xb
149 #define RESET_DEVICE                    0xd
150
151 /*
152  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
153  * the driver calls the chunks 'step' or 'codeword' interchangeably
154  */
155 #define NANDC_STEP_SIZE                 512
156
157 /*
158  * the largest page size we support is 8K, this will have 16 steps/codewords
159  * of 512 bytes each
160  */
161 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
162
163 /* we read at most 3 registers per codeword scan */
164 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
165
166 /* ECC modes supported by the controller */
167 #define ECC_NONE        BIT(0)
168 #define ECC_RS_4BIT     BIT(1)
169 #define ECC_BCH_4BIT    BIT(2)
170 #define ECC_BCH_8BIT    BIT(3)
171
172 struct desc_info {
173         struct list_head node;
174
175         enum dma_data_direction dir;
176         struct scatterlist sgl;
177         struct dma_async_tx_descriptor *dma_desc;
178 };
179
180 /*
181  * holds the current register values that we want to write. acts as a contiguous
182  * chunk of memory which we use to write the controller registers through DMA.
183  */
184 struct nandc_regs {
185         __le32 cmd;
186         __le32 addr0;
187         __le32 addr1;
188         __le32 chip_sel;
189         __le32 exec;
190
191         __le32 cfg0;
192         __le32 cfg1;
193         __le32 ecc_bch_cfg;
194
195         __le32 clrflashstatus;
196         __le32 clrreadstatus;
197
198         __le32 cmd1;
199         __le32 vld;
200
201         __le32 orig_cmd1;
202         __le32 orig_vld;
203
204         __le32 ecc_buf_cfg;
205 };
206
207 /*
208  * NAND controller data struct
209  *
210  * @controller:                 base controller structure
211  * @host_list:                  list containing all the chips attached to the
212  *                              controller
213  * @dev:                        parent device
214  * @base:                       MMIO base
215  * @base_dma:                   physical base address of controller registers
216  * @core_clk:                   controller clock
217  * @aon_clk:                    another controller clock
218  *
219  * @chan:                       dma channel
220  * @cmd_crci:                   ADM DMA CRCI for command flow control
221  * @data_crci:                  ADM DMA CRCI for data flow control
222  * @desc_list:                  DMA descriptor list (list of desc_infos)
223  *
224  * @data_buffer:                our local DMA buffer for page read/writes,
225  *                              used when we can't use the buffer provided
226  *                              by upper layers directly
227  * @buf_size/count/start:       markers for chip->read_buf/write_buf functions
228  * @reg_read_buf:               local buffer for reading back registers via DMA
229  * @reg_read_pos:               marker for data read in reg_read_buf
230  *
231  * @regs:                       a contiguous chunk of memory for DMA register
232  *                              writes. contains the register values to be
233  *                              written to controller
234  * @cmd1/vld:                   some fixed controller register values
235  * @ecc_modes:                  supported ECC modes by the current controller,
236  *                              initialized via DT match data
237  */
238 struct qcom_nand_controller {
239         struct nand_hw_control controller;
240         struct list_head host_list;
241
242         struct device *dev;
243
244         void __iomem *base;
245         dma_addr_t base_dma;
246
247         struct clk *core_clk;
248         struct clk *aon_clk;
249
250         struct dma_chan *chan;
251         unsigned int cmd_crci;
252         unsigned int data_crci;
253         struct list_head desc_list;
254
255         u8              *data_buffer;
256         int             buf_size;
257         int             buf_count;
258         int             buf_start;
259
260         __le32 *reg_read_buf;
261         int reg_read_pos;
262
263         struct nandc_regs *regs;
264
265         u32 cmd1, vld;
266         u32 ecc_modes;
267 };
268
269 /*
270  * NAND chip structure
271  *
272  * @chip:                       base NAND chip structure
273  * @node:                       list node to add itself to host_list in
274  *                              qcom_nand_controller
275  *
276  * @cs:                         chip select value for this chip
277  * @cw_size:                    the number of bytes in a single step/codeword
278  *                              of a page, consisting of all data, ecc, spare
279  *                              and reserved bytes
280  * @cw_data:                    the number of bytes within a codeword protected
281  *                              by ECC
282  * @use_ecc:                    request the controller to use ECC for the
283  *                              upcoming read/write
284  * @bch_enabled:                flag to tell whether BCH ECC mode is used
285  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
286  *                              chip
287  * @status:                     value to be returned if NAND_CMD_STATUS command
288  *                              is executed
289  * @last_command:               keeps track of last command on this chip. used
290  *                              for reading correct status
291  *
292  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
293  *                              ecc/non-ecc mode for the current nand flash
294  *                              device
295  */
296 struct qcom_nand_host {
297         struct nand_chip chip;
298         struct list_head node;
299
300         int cs;
301         int cw_size;
302         int cw_data;
303         bool use_ecc;
304         bool bch_enabled;
305         int ecc_bytes_hw;
306         int spare_bytes;
307         int bbm_size;
308         u8 status;
309         int last_command;
310
311         u32 cfg0, cfg1;
312         u32 cfg0_raw, cfg1_raw;
313         u32 ecc_buf_cfg;
314         u32 ecc_bch_cfg;
315         u32 clrflashstatus;
316         u32 clrreadstatus;
317 };
318
319 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
320 {
321         return container_of(chip, struct qcom_nand_host, chip);
322 }
323
324 static inline struct qcom_nand_controller *
325 get_qcom_nand_controller(struct nand_chip *chip)
326 {
327         return container_of(chip->controller, struct qcom_nand_controller,
328                             controller);
329 }
330
331 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
332 {
333         return ioread32(nandc->base + offset);
334 }
335
336 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
337                                u32 val)
338 {
339         iowrite32(val, nandc->base + offset);
340 }
341
342 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
343 {
344         switch (offset) {
345         case NAND_FLASH_CMD:
346                 return &regs->cmd;
347         case NAND_ADDR0:
348                 return &regs->addr0;
349         case NAND_ADDR1:
350                 return &regs->addr1;
351         case NAND_FLASH_CHIP_SELECT:
352                 return &regs->chip_sel;
353         case NAND_EXEC_CMD:
354                 return &regs->exec;
355         case NAND_FLASH_STATUS:
356                 return &regs->clrflashstatus;
357         case NAND_DEV0_CFG0:
358                 return &regs->cfg0;
359         case NAND_DEV0_CFG1:
360                 return &regs->cfg1;
361         case NAND_DEV0_ECC_CFG:
362                 return &regs->ecc_bch_cfg;
363         case NAND_READ_STATUS:
364                 return &regs->clrreadstatus;
365         case NAND_DEV_CMD1:
366                 return &regs->cmd1;
367         case NAND_DEV_CMD1_RESTORE:
368                 return &regs->orig_cmd1;
369         case NAND_DEV_CMD_VLD:
370                 return &regs->vld;
371         case NAND_DEV_CMD_VLD_RESTORE:
372                 return &regs->orig_vld;
373         case NAND_EBI2_ECC_BUF_CFG:
374                 return &regs->ecc_buf_cfg;
375         default:
376                 return NULL;
377         }
378 }
379
380 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
381                           u32 val)
382 {
383         struct nandc_regs *regs = nandc->regs;
384         __le32 *reg;
385
386         reg = offset_to_nandc_reg(regs, offset);
387
388         if (reg)
389                 *reg = cpu_to_le32(val);
390 }
391
392 /* helper to configure address register values */
393 static void set_address(struct qcom_nand_host *host, u16 column, int page)
394 {
395         struct nand_chip *chip = &host->chip;
396         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
397
398         if (chip->options & NAND_BUSWIDTH_16)
399                 column >>= 1;
400
401         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
402         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
403 }
404
405 /*
406  * update_rw_regs:      set up read/write register values, these will be
407  *                      written to the NAND controller registers via DMA
408  *
409  * @num_cw:             number of steps for the read/write operation
410  * @read:               read or write operation
411  */
412 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
413 {
414         struct nand_chip *chip = &host->chip;
415         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
416         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
417
418         if (read) {
419                 if (host->use_ecc)
420                         cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
421                 else
422                         cmd = PAGE_READ | PAGE_ACC | LAST_PAGE;
423         } else {
424                         cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
425         }
426
427         if (host->use_ecc) {
428                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
429                                 (num_cw - 1) << CW_PER_PAGE;
430
431                 cfg1 = host->cfg1;
432                 ecc_bch_cfg = host->ecc_bch_cfg;
433         } else {
434                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
435                                 (num_cw - 1) << CW_PER_PAGE;
436
437                 cfg1 = host->cfg1_raw;
438                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
439         }
440
441         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
442         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
443         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
444         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
445         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
446         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
447         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
448         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
449 }
450
451 static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
452                          int reg_off, const void *vaddr, int size,
453                          bool flow_control)
454 {
455         struct desc_info *desc;
456         struct dma_async_tx_descriptor *dma_desc;
457         struct scatterlist *sgl;
458         struct dma_slave_config slave_conf;
459         enum dma_transfer_direction dir_eng;
460         int ret;
461
462         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
463         if (!desc)
464                 return -ENOMEM;
465
466         sgl = &desc->sgl;
467
468         sg_init_one(sgl, vaddr, size);
469
470         if (read) {
471                 dir_eng = DMA_DEV_TO_MEM;
472                 desc->dir = DMA_FROM_DEVICE;
473         } else {
474                 dir_eng = DMA_MEM_TO_DEV;
475                 desc->dir = DMA_TO_DEVICE;
476         }
477
478         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
479         if (ret == 0) {
480                 ret = -ENOMEM;
481                 goto err;
482         }
483
484         memset(&slave_conf, 0x00, sizeof(slave_conf));
485
486         slave_conf.device_fc = flow_control;
487         if (read) {
488                 slave_conf.src_maxburst = 16;
489                 slave_conf.src_addr = nandc->base_dma + reg_off;
490                 slave_conf.slave_id = nandc->data_crci;
491         } else {
492                 slave_conf.dst_maxburst = 16;
493                 slave_conf.dst_addr = nandc->base_dma + reg_off;
494                 slave_conf.slave_id = nandc->cmd_crci;
495         }
496
497         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
498         if (ret) {
499                 dev_err(nandc->dev, "failed to configure dma channel\n");
500                 goto err;
501         }
502
503         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
504         if (!dma_desc) {
505                 dev_err(nandc->dev, "failed to prepare desc\n");
506                 ret = -EINVAL;
507                 goto err;
508         }
509
510         desc->dma_desc = dma_desc;
511
512         list_add_tail(&desc->node, &nandc->desc_list);
513
514         return 0;
515 err:
516         kfree(desc);
517
518         return ret;
519 }
520
521 /*
522  * read_reg_dma:        prepares a descriptor to read a given number of
523  *                      contiguous registers to the reg_read_buf pointer
524  *
525  * @first:              offset of the first register in the contiguous block
526  * @num_regs:           number of registers to read
527  */
528 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
529                         int num_regs)
530 {
531         bool flow_control = false;
532         void *vaddr;
533         int size;
534
535         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
536                 flow_control = true;
537
538         size = num_regs * sizeof(u32);
539         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
540         nandc->reg_read_pos += num_regs;
541
542         return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
543 }
544
545 /*
546  * write_reg_dma:       prepares a descriptor to write a given number of
547  *                      contiguous registers
548  *
549  * @first:              offset of the first register in the contiguous block
550  * @num_regs:           number of registers to write
551  */
552 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
553                          int num_regs)
554 {
555         bool flow_control = false;
556         struct nandc_regs *regs = nandc->regs;
557         void *vaddr;
558         int size;
559
560         vaddr = offset_to_nandc_reg(regs, first);
561
562         if (first == NAND_FLASH_CMD)
563                 flow_control = true;
564
565         if (first == NAND_DEV_CMD1_RESTORE)
566                 first = NAND_DEV_CMD1;
567
568         if (first == NAND_DEV_CMD_VLD_RESTORE)
569                 first = NAND_DEV_CMD_VLD;
570
571         size = num_regs * sizeof(u32);
572
573         return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
574 }
575
576 /*
577  * read_data_dma:       prepares a DMA descriptor to transfer data from the
578  *                      controller's internal buffer to the buffer 'vaddr'
579  *
580  * @reg_off:            offset within the controller's data buffer
581  * @vaddr:              virtual address of the buffer we want to write to
582  * @size:               DMA transaction size in bytes
583  */
584 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
585                          const u8 *vaddr, int size)
586 {
587         return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
588 }
589
590 /*
591  * write_data_dma:      prepares a DMA descriptor to transfer data from
592  *                      'vaddr' to the controller's internal buffer
593  *
594  * @reg_off:            offset within the controller's data buffer
595  * @vaddr:              virtual address of the buffer we want to read from
596  * @size:               DMA transaction size in bytes
597  */
598 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
599                           const u8 *vaddr, int size)
600 {
601         return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
602 }
603
604 /*
605  * helper to prepare dma descriptors to configure registers needed for reading a
606  * codeword/step in a page
607  */
608 static void config_cw_read(struct qcom_nand_controller *nandc)
609 {
610         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
611         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
612         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
613
614         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
615
616         read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
617         read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
618 }
619
620 /*
621  * helpers to prepare dma descriptors used to configure registers needed for
622  * writing a codeword/step in a page
623  */
624 static void config_cw_write_pre(struct qcom_nand_controller *nandc)
625 {
626         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
627         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
628         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
629 }
630
631 static void config_cw_write_post(struct qcom_nand_controller *nandc)
632 {
633         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
634
635         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
636
637         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
638         write_reg_dma(nandc, NAND_READ_STATUS, 1);
639 }
640
641 /*
642  * the following functions are used within chip->cmdfunc() to perform different
643  * NAND_CMD_* commands
644  */
645
646 /* sets up descriptors for NAND_CMD_PARAM */
647 static int nandc_param(struct qcom_nand_host *host)
648 {
649         struct nand_chip *chip = &host->chip;
650         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
651
652         /*
653          * NAND_CMD_PARAM is called before we know much about the FLASH chip
654          * in use. we configure the controller to perform a raw read of 512
655          * bytes to read onfi params
656          */
657         nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE);
658         nandc_set_reg(nandc, NAND_ADDR0, 0);
659         nandc_set_reg(nandc, NAND_ADDR1, 0);
660         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
661                                         | 512 << UD_SIZE_BYTES
662                                         | 5 << NUM_ADDR_CYCLES
663                                         | 0 << SPARE_SIZE_BYTES);
664         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
665                                         | 0 << CS_ACTIVE_BSY
666                                         | 17 << BAD_BLOCK_BYTE_NUM
667                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
668                                         | 2 << WR_RD_BSY_GAP
669                                         | 0 << WIDE_FLASH
670                                         | 1 << DEV0_CFG1_ECC_DISABLE);
671         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
672
673         /* configure CMD1 and VLD for ONFI param probing */
674         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
675                       (nandc->vld & ~(1 << READ_START_VLD))
676                       | 0 << READ_START_VLD);
677         nandc_set_reg(nandc, NAND_DEV_CMD1,
678                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
679                       | NAND_CMD_PARAM << READ_ADDR);
680
681         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
682
683         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
684         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
685
686         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
687         write_reg_dma(nandc, NAND_DEV_CMD1, 1);
688
689         nandc->buf_count = 512;
690         memset(nandc->data_buffer, 0xff, nandc->buf_count);
691
692         config_cw_read(nandc);
693
694         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
695                       nandc->buf_count);
696
697         /* restore CMD1 and VLD regs */
698         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
699         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);
700
701         return 0;
702 }
703
704 /* sets up descriptors for NAND_CMD_ERASE1 */
705 static int erase_block(struct qcom_nand_host *host, int page_addr)
706 {
707         struct nand_chip *chip = &host->chip;
708         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
709
710         nandc_set_reg(nandc, NAND_FLASH_CMD,
711                       BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
712         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
713         nandc_set_reg(nandc, NAND_ADDR1, 0);
714         nandc_set_reg(nandc, NAND_DEV0_CFG0,
715                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
716         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
717         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
718         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
719         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
720
721         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
722         write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
723         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
724
725         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
726
727         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
728         write_reg_dma(nandc, NAND_READ_STATUS, 1);
729
730         return 0;
731 }
732
733 /* sets up descriptors for NAND_CMD_READID */
734 static int read_id(struct qcom_nand_host *host, int column)
735 {
736         struct nand_chip *chip = &host->chip;
737         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
738
739         if (column == -1)
740                 return 0;
741
742         nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
743         nandc_set_reg(nandc, NAND_ADDR0, column);
744         nandc_set_reg(nandc, NAND_ADDR1, 0);
745         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
746         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
747
748         write_reg_dma(nandc, NAND_FLASH_CMD, 4);
749         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
750
751         read_reg_dma(nandc, NAND_READ_ID, 1);
752
753         return 0;
754 }
755
756 /* sets up descriptors for NAND_CMD_RESET */
757 static int reset(struct qcom_nand_host *host)
758 {
759         struct nand_chip *chip = &host->chip;
760         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
761
762         nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
763         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
764
765         write_reg_dma(nandc, NAND_FLASH_CMD, 1);
766         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
767
768         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
769
770         return 0;
771 }
772
773 /* helpers to submit/free our list of dma descriptors */
774 static int submit_descs(struct qcom_nand_controller *nandc)
775 {
776         struct desc_info *desc;
777         dma_cookie_t cookie = 0;
778
779         list_for_each_entry(desc, &nandc->desc_list, node)
780                 cookie = dmaengine_submit(desc->dma_desc);
781
782         if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
783                 return -ETIMEDOUT;
784
785         return 0;
786 }
787
788 static void free_descs(struct qcom_nand_controller *nandc)
789 {
790         struct desc_info *desc, *n;
791
792         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
793                 list_del(&desc->node);
794                 dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
795                 kfree(desc);
796         }
797 }
798
799 /* reset the register read buffer for next NAND operation */
800 static void clear_read_regs(struct qcom_nand_controller *nandc)
801 {
802         nandc->reg_read_pos = 0;
803         memset(nandc->reg_read_buf, 0,
804                MAX_REG_RD * sizeof(*nandc->reg_read_buf));
805 }
806
807 static void pre_command(struct qcom_nand_host *host, int command)
808 {
809         struct nand_chip *chip = &host->chip;
810         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
811
812         nandc->buf_count = 0;
813         nandc->buf_start = 0;
814         host->use_ecc = false;
815         host->last_command = command;
816
817         clear_read_regs(nandc);
818 }
819
820 /*
821  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
822  * privately maintained status byte, this status byte can be read after
823  * NAND_CMD_STATUS is called
824  */
825 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
826 {
827         struct nand_chip *chip = &host->chip;
828         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
829         struct nand_ecc_ctrl *ecc = &chip->ecc;
830         int num_cw;
831         int i;
832
833         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
834
835         for (i = 0; i < num_cw; i++) {
836                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
837
838                 if (flash_status & FS_MPU_ERR)
839                         host->status &= ~NAND_STATUS_WP;
840
841                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
842                                                  (flash_status &
843                                                   FS_DEVICE_STS_ERR)))
844                         host->status |= NAND_STATUS_FAIL;
845         }
846 }
847
848 static void post_command(struct qcom_nand_host *host, int command)
849 {
850         struct nand_chip *chip = &host->chip;
851         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
852
853         switch (command) {
854         case NAND_CMD_READID:
855                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
856                        nandc->buf_count);
857                 break;
858         case NAND_CMD_PAGEPROG:
859         case NAND_CMD_ERASE1:
860                 parse_erase_write_errors(host, command);
861                 break;
862         default:
863                 break;
864         }
865 }
866
867 /*
868  * Implements chip->cmdfunc. It's  only used for a limited set of commands.
869  * The rest of the commands wouldn't be called by upper layers. For example,
870  * NAND_CMD_READOOB would never be called because we have our own versions
871  * of read_oob ops for nand_ecc_ctrl.
872  */
873 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
874                                int column, int page_addr)
875 {
876         struct nand_chip *chip = mtd_to_nand(mtd);
877         struct qcom_nand_host *host = to_qcom_nand_host(chip);
878         struct nand_ecc_ctrl *ecc = &chip->ecc;
879         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
880         bool wait = false;
881         int ret = 0;
882
883         pre_command(host, command);
884
885         switch (command) {
886         case NAND_CMD_RESET:
887                 ret = reset(host);
888                 wait = true;
889                 break;
890
891         case NAND_CMD_READID:
892                 nandc->buf_count = 4;
893                 ret = read_id(host, column);
894                 wait = true;
895                 break;
896
897         case NAND_CMD_PARAM:
898                 ret = nandc_param(host);
899                 wait = true;
900                 break;
901
902         case NAND_CMD_ERASE1:
903                 ret = erase_block(host, page_addr);
904                 wait = true;
905                 break;
906
907         case NAND_CMD_READ0:
908                 /* we read the entire page for now */
909                 WARN_ON(column != 0);
910
911                 host->use_ecc = true;
912                 set_address(host, 0, page_addr);
913                 update_rw_regs(host, ecc->steps, true);
914                 break;
915
916         case NAND_CMD_SEQIN:
917                 WARN_ON(column != 0);
918                 set_address(host, 0, page_addr);
919                 break;
920
921         case NAND_CMD_PAGEPROG:
922         case NAND_CMD_STATUS:
923         case NAND_CMD_NONE:
924         default:
925                 break;
926         }
927
928         if (ret) {
929                 dev_err(nandc->dev, "failure executing command %d\n",
930                         command);
931                 free_descs(nandc);
932                 return;
933         }
934
935         if (wait) {
936                 ret = submit_descs(nandc);
937                 if (ret)
938                         dev_err(nandc->dev,
939                                 "failure submitting descs for command %d\n",
940                                 command);
941         }
942
943         free_descs(nandc);
944
945         post_command(host, command);
946 }
947
948 /*
949  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
950  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
951  *
952  * when using RS ECC, the HW reports the same erros when reading an erased CW,
953  * but it notifies that it is an erased CW by placing special characters at
954  * certain offsets in the buffer.
955  *
956  * verify if the page is erased or not, and fix up the page for RS ECC by
957  * replacing the special characters with 0xff.
958  */
959 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
960 {
961         u8 empty1, empty2;
962
963         /*
964          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
965          * is erased by looking for 0x54s at offsets 3 and 175 from the
966          * beginning of each codeword
967          */
968
969         empty1 = data_buf[3];
970         empty2 = data_buf[175];
971
972         /*
973          * if the erased codework markers, if they exist override them with
974          * 0xffs
975          */
976         if ((empty1 == 0x54 && empty2 == 0xff) ||
977             (empty1 == 0xff && empty2 == 0x54)) {
978                 data_buf[3] = 0xff;
979                 data_buf[175] = 0xff;
980         }
981
982         /*
983          * check if the entire chunk contains 0xffs or not. if it doesn't, then
984          * restore the original values at the special offsets
985          */
986         if (memchr_inv(data_buf, 0xff, data_len)) {
987                 data_buf[3] = empty1;
988                 data_buf[175] = empty2;
989
990                 return false;
991         }
992
993         return true;
994 }
995
996 struct read_stats {
997         __le32 flash;
998         __le32 buffer;
999         __le32 erased_cw;
1000 };
1001
1002 /*
1003  * reads back status registers set by the controller to notify page read
1004  * errors. this is equivalent to what 'ecc->correct()' would do.
1005  */
1006 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1007                              u8 *oob_buf)
1008 {
1009         struct nand_chip *chip = &host->chip;
1010         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1011         struct mtd_info *mtd = nand_to_mtd(chip);
1012         struct nand_ecc_ctrl *ecc = &chip->ecc;
1013         unsigned int max_bitflips = 0;
1014         struct read_stats *buf;
1015         int i;
1016
1017         buf = (struct read_stats *)nandc->reg_read_buf;
1018
1019         for (i = 0; i < ecc->steps; i++, buf++) {
1020                 u32 flash, buffer, erased_cw;
1021                 int data_len, oob_len;
1022
1023                 if (i == (ecc->steps - 1)) {
1024                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1025                         oob_len = ecc->steps << 2;
1026                 } else {
1027                         data_len = host->cw_data;
1028                         oob_len = 0;
1029                 }
1030
1031                 flash = le32_to_cpu(buf->flash);
1032                 buffer = le32_to_cpu(buf->buffer);
1033                 erased_cw = le32_to_cpu(buf->erased_cw);
1034
1035                 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1036                         bool erased;
1037
1038                         /* ignore erased codeword errors */
1039                         if (host->bch_enabled) {
1040                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1041                                          true : false;
1042                         } else {
1043                                 erased = erased_chunk_check_and_fixup(data_buf,
1044                                                                       data_len);
1045                         }
1046
1047                         if (erased) {
1048                                 data_buf += data_len;
1049                                 if (oob_buf)
1050                                         oob_buf += oob_len + ecc->bytes;
1051                                 continue;
1052                         }
1053
1054                         if (buffer & BS_UNCORRECTABLE_BIT) {
1055                                 int ret, ecclen, extraooblen;
1056                                 void *eccbuf;
1057
1058                                 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1059                                 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1060                                 extraooblen = oob_buf ? oob_len : 0;
1061
1062                                 /*
1063                                  * make sure it isn't an erased page reported
1064                                  * as not-erased by HW because of a few bitflips
1065                                  */
1066                                 ret = nand_check_erased_ecc_chunk(data_buf,
1067                                         data_len, eccbuf, ecclen, oob_buf,
1068                                         extraooblen, ecc->strength);
1069                                 if (ret < 0) {
1070                                         mtd->ecc_stats.failed++;
1071                                 } else {
1072                                         mtd->ecc_stats.corrected += ret;
1073                                         max_bitflips =
1074                                                 max_t(unsigned int, max_bitflips, ret);
1075                                 }
1076                         }
1077                 } else {
1078                         unsigned int stat;
1079
1080                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1081                         mtd->ecc_stats.corrected += stat;
1082                         max_bitflips = max(max_bitflips, stat);
1083                 }
1084
1085                 data_buf += data_len;
1086                 if (oob_buf)
1087                         oob_buf += oob_len + ecc->bytes;
1088         }
1089
1090         return max_bitflips;
1091 }
1092
1093 /*
1094  * helper to perform the actual page read operation, used by ecc->read_page(),
1095  * ecc->read_oob()
1096  */
1097 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1098                          u8 *oob_buf)
1099 {
1100         struct nand_chip *chip = &host->chip;
1101         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1102         struct nand_ecc_ctrl *ecc = &chip->ecc;
1103         int i, ret;
1104
1105         /* queue cmd descs for each codeword */
1106         for (i = 0; i < ecc->steps; i++) {
1107                 int data_size, oob_size;
1108
1109                 if (i == (ecc->steps - 1)) {
1110                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1111                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1112                                    host->spare_bytes;
1113                 } else {
1114                         data_size = host->cw_data;
1115                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1116                 }
1117
1118                 config_cw_read(nandc);
1119
1120                 if (data_buf)
1121                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1122                                       data_size);
1123
1124                 /*
1125                  * when ecc is enabled, the controller doesn't read the real
1126                  * or dummy bad block markers in each chunk. To maintain a
1127                  * consistent layout across RAW and ECC reads, we just
1128                  * leave the real/dummy BBM offsets empty (i.e, filled with
1129                  * 0xffs)
1130                  */
1131                 if (oob_buf) {
1132                         int j;
1133
1134                         for (j = 0; j < host->bbm_size; j++)
1135                                 *oob_buf++ = 0xff;
1136
1137                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1138                                       oob_buf, oob_size);
1139                 }
1140
1141                 if (data_buf)
1142                         data_buf += data_size;
1143                 if (oob_buf)
1144                         oob_buf += oob_size;
1145         }
1146
1147         ret = submit_descs(nandc);
1148         if (ret)
1149                 dev_err(nandc->dev, "failure to read page/oob\n");
1150
1151         free_descs(nandc);
1152
1153         return ret;
1154 }
1155
1156 /*
1157  * a helper that copies the last step/codeword of a page (containing free oob)
1158  * into our local buffer
1159  */
1160 static int copy_last_cw(struct qcom_nand_host *host, int page)
1161 {
1162         struct nand_chip *chip = &host->chip;
1163         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1164         struct nand_ecc_ctrl *ecc = &chip->ecc;
1165         int size;
1166         int ret;
1167
1168         clear_read_regs(nandc);
1169
1170         size = host->use_ecc ? host->cw_data : host->cw_size;
1171
1172         /* prepare a clean read buffer */
1173         memset(nandc->data_buffer, 0xff, size);
1174
1175         set_address(host, host->cw_size * (ecc->steps - 1), page);
1176         update_rw_regs(host, 1, true);
1177
1178         config_cw_read(nandc);
1179
1180         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);
1181
1182         ret = submit_descs(nandc);
1183         if (ret)
1184                 dev_err(nandc->dev, "failed to copy last codeword\n");
1185
1186         free_descs(nandc);
1187
1188         return ret;
1189 }
1190
1191 /* implements ecc->read_page() */
1192 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1193                                 uint8_t *buf, int oob_required, int page)
1194 {
1195         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1196         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1197         u8 *data_buf, *oob_buf = NULL;
1198         int ret;
1199
1200         data_buf = buf;
1201         oob_buf = oob_required ? chip->oob_poi : NULL;
1202
1203         ret = read_page_ecc(host, data_buf, oob_buf);
1204         if (ret) {
1205                 dev_err(nandc->dev, "failure to read page\n");
1206                 return ret;
1207         }
1208
1209         return parse_read_errors(host, data_buf, oob_buf);
1210 }
1211
1212 /* implements ecc->read_page_raw() */
1213 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1214                                     struct nand_chip *chip, uint8_t *buf,
1215                                     int oob_required, int page)
1216 {
1217         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1218         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1219         u8 *data_buf, *oob_buf;
1220         struct nand_ecc_ctrl *ecc = &chip->ecc;
1221         int i, ret;
1222
1223         data_buf = buf;
1224         oob_buf = chip->oob_poi;
1225
1226         host->use_ecc = false;
1227         update_rw_regs(host, ecc->steps, true);
1228
1229         for (i = 0; i < ecc->steps; i++) {
1230                 int data_size1, data_size2, oob_size1, oob_size2;
1231                 int reg_off = FLASH_BUF_ACC;
1232
1233                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1234                 oob_size1 = host->bbm_size;
1235
1236                 if (i == (ecc->steps - 1)) {
1237                         data_size2 = ecc->size - data_size1 -
1238                                      ((ecc->steps - 1) << 2);
1239                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1240                                     host->spare_bytes;
1241                 } else {
1242                         data_size2 = host->cw_data - data_size1;
1243                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1244                 }
1245
1246                 config_cw_read(nandc);
1247
1248                 read_data_dma(nandc, reg_off, data_buf, data_size1);
1249                 reg_off += data_size1;
1250                 data_buf += data_size1;
1251
1252                 read_data_dma(nandc, reg_off, oob_buf, oob_size1);
1253                 reg_off += oob_size1;
1254                 oob_buf += oob_size1;
1255
1256                 read_data_dma(nandc, reg_off, data_buf, data_size2);
1257                 reg_off += data_size2;
1258                 data_buf += data_size2;
1259
1260                 read_data_dma(nandc, reg_off, oob_buf, oob_size2);
1261                 oob_buf += oob_size2;
1262         }
1263
1264         ret = submit_descs(nandc);
1265         if (ret)
1266                 dev_err(nandc->dev, "failure to read raw page\n");
1267
1268         free_descs(nandc);
1269
1270         return 0;
1271 }
1272
1273 /* implements ecc->read_oob() */
1274 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1275                                int page)
1276 {
1277         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1278         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1279         struct nand_ecc_ctrl *ecc = &chip->ecc;
1280         int ret;
1281
1282         clear_read_regs(nandc);
1283
1284         host->use_ecc = true;
1285         set_address(host, 0, page);
1286         update_rw_regs(host, ecc->steps, true);
1287
1288         ret = read_page_ecc(host, NULL, chip->oob_poi);
1289         if (ret)
1290                 dev_err(nandc->dev, "failure to read oob\n");
1291
1292         return ret;
1293 }
1294
1295 /* implements ecc->write_page() */
1296 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1297                                  const uint8_t *buf, int oob_required, int page)
1298 {
1299         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1300         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1301         struct nand_ecc_ctrl *ecc = &chip->ecc;
1302         u8 *data_buf, *oob_buf;
1303         int i, ret;
1304
1305         clear_read_regs(nandc);
1306
1307         data_buf = (u8 *)buf;
1308         oob_buf = chip->oob_poi;
1309
1310         host->use_ecc = true;
1311         update_rw_regs(host, ecc->steps, false);
1312
1313         for (i = 0; i < ecc->steps; i++) {
1314                 int data_size, oob_size;
1315
1316                 if (i == (ecc->steps - 1)) {
1317                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1318                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1319                                    host->spare_bytes;
1320                 } else {
1321                         data_size = host->cw_data;
1322                         oob_size = ecc->bytes;
1323                 }
1324
1325                 config_cw_write_pre(nandc);
1326
1327                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);
1328
1329                 /*
1330                  * when ECC is enabled, we don't really need to write anything
1331                  * to oob for the first n - 1 codewords since these oob regions
1332                  * just contain ECC bytes that's written by the controller
1333                  * itself. For the last codeword, we skip the bbm positions and
1334                  * write to the free oob area.
1335                  */
1336                 if (i == (ecc->steps - 1)) {
1337                         oob_buf += host->bbm_size;
1338
1339                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1340                                        oob_buf, oob_size);
1341                 }
1342
1343                 config_cw_write_post(nandc);
1344
1345                 data_buf += data_size;
1346                 oob_buf += oob_size;
1347         }
1348
1349         ret = submit_descs(nandc);
1350         if (ret)
1351                 dev_err(nandc->dev, "failure to write page\n");
1352
1353         free_descs(nandc);
1354
1355         return ret;
1356 }
1357
1358 /* implements ecc->write_page_raw() */
1359 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1360                                      struct nand_chip *chip, const uint8_t *buf,
1361                                      int oob_required, int page)
1362 {
1363         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1364         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1365         struct nand_ecc_ctrl *ecc = &chip->ecc;
1366         u8 *data_buf, *oob_buf;
1367         int i, ret;
1368
1369         clear_read_regs(nandc);
1370
1371         data_buf = (u8 *)buf;
1372         oob_buf = chip->oob_poi;
1373
1374         host->use_ecc = false;
1375         update_rw_regs(host, ecc->steps, false);
1376
1377         for (i = 0; i < ecc->steps; i++) {
1378                 int data_size1, data_size2, oob_size1, oob_size2;
1379                 int reg_off = FLASH_BUF_ACC;
1380
1381                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1382                 oob_size1 = host->bbm_size;
1383
1384                 if (i == (ecc->steps - 1)) {
1385                         data_size2 = ecc->size - data_size1 -
1386                                      ((ecc->steps - 1) << 2);
1387                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1388                                     host->spare_bytes;
1389                 } else {
1390                         data_size2 = host->cw_data - data_size1;
1391                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1392                 }
1393
1394                 config_cw_write_pre(nandc);
1395
1396                 write_data_dma(nandc, reg_off, data_buf, data_size1);
1397                 reg_off += data_size1;
1398                 data_buf += data_size1;
1399
1400                 write_data_dma(nandc, reg_off, oob_buf, oob_size1);
1401                 reg_off += oob_size1;
1402                 oob_buf += oob_size1;
1403
1404                 write_data_dma(nandc, reg_off, data_buf, data_size2);
1405                 reg_off += data_size2;
1406                 data_buf += data_size2;
1407
1408                 write_data_dma(nandc, reg_off, oob_buf, oob_size2);
1409                 oob_buf += oob_size2;
1410
1411                 config_cw_write_post(nandc);
1412         }
1413
1414         ret = submit_descs(nandc);
1415         if (ret)
1416                 dev_err(nandc->dev, "failure to write raw page\n");
1417
1418         free_descs(nandc);
1419
1420         return ret;
1421 }
1422
1423 /*
1424  * implements ecc->write_oob()
1425  *
1426  * the NAND controller cannot write only data or only oob within a codeword,
1427  * since ecc is calculated for the combined codeword. we first copy the
1428  * entire contents for the last codeword(data + oob), replace the old oob
1429  * with the new one in chip->oob_poi, and then write the entire codeword.
1430  * this read-copy-write operation results in a slight performance loss.
1431  */
1432 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1433                                 int page)
1434 {
1435         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1436         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1437         struct nand_ecc_ctrl *ecc = &chip->ecc;
1438         u8 *oob = chip->oob_poi;
1439         int data_size, oob_size;
1440         int ret, status = 0;
1441
1442         host->use_ecc = true;
1443
1444         ret = copy_last_cw(host, page);
1445         if (ret)
1446                 return ret;
1447
1448         clear_read_regs(nandc);
1449
1450         /* calculate the data and oob size for the last codeword/step */
1451         data_size = ecc->size - ((ecc->steps - 1) << 2);
1452         oob_size = mtd->oobavail;
1453
1454         /* override new oob content to last codeword */
1455         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1456                                     0, mtd->oobavail);
1457
1458         set_address(host, host->cw_size * (ecc->steps - 1), page);
1459         update_rw_regs(host, 1, false);
1460
1461         config_cw_write_pre(nandc);
1462         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1463                        data_size + oob_size);
1464         config_cw_write_post(nandc);
1465
1466         ret = submit_descs(nandc);
1467
1468         free_descs(nandc);
1469
1470         if (ret) {
1471                 dev_err(nandc->dev, "failure to write oob\n");
1472                 return -EIO;
1473         }
1474
1475         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1476
1477         status = chip->waitfunc(mtd, chip);
1478
1479         return status & NAND_STATUS_FAIL ? -EIO : 0;
1480 }
1481
1482 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1483 {
1484         struct nand_chip *chip = mtd_to_nand(mtd);
1485         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1486         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1487         struct nand_ecc_ctrl *ecc = &chip->ecc;
1488         int page, ret, bbpos, bad = 0;
1489         u32 flash_status;
1490
1491         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1492
1493         /*
1494          * configure registers for a raw sub page read, the address is set to
1495          * the beginning of the last codeword, we don't care about reading ecc
1496          * portion of oob. we just want the first few bytes from this codeword
1497          * that contains the BBM
1498          */
1499         host->use_ecc = false;
1500
1501         ret = copy_last_cw(host, page);
1502         if (ret)
1503                 goto err;
1504
1505         flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1506
1507         if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1508                 dev_warn(nandc->dev, "error when trying to read BBM\n");
1509                 goto err;
1510         }
1511
1512         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1513
1514         bad = nandc->data_buffer[bbpos] != 0xff;
1515
1516         if (chip->options & NAND_BUSWIDTH_16)
1517                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1518 err:
1519         return bad;
1520 }
1521
1522 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1523 {
1524         struct nand_chip *chip = mtd_to_nand(mtd);
1525         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1526         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1527         struct nand_ecc_ctrl *ecc = &chip->ecc;
1528         int page, ret, status = 0;
1529
1530         clear_read_regs(nandc);
1531
1532         /*
1533          * to mark the BBM as bad, we flash the entire last codeword with 0s.
1534          * we don't care about the rest of the content in the codeword since
1535          * we aren't going to use this block again
1536          */
1537         memset(nandc->data_buffer, 0x00, host->cw_size);
1538
1539         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1540
1541         /* prepare write */
1542         host->use_ecc = false;
1543         set_address(host, host->cw_size * (ecc->steps - 1), page);
1544         update_rw_regs(host, 1, false);
1545
1546         config_cw_write_pre(nandc);
1547         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
1548         config_cw_write_post(nandc);
1549
1550         ret = submit_descs(nandc);
1551
1552         free_descs(nandc);
1553
1554         if (ret) {
1555                 dev_err(nandc->dev, "failure to update BBM\n");
1556                 return -EIO;
1557         }
1558
1559         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1560
1561         status = chip->waitfunc(mtd, chip);
1562
1563         return status & NAND_STATUS_FAIL ? -EIO : 0;
1564 }
1565
1566 /*
1567  * the three functions below implement chip->read_byte(), chip->read_buf()
1568  * and chip->write_buf() respectively. these aren't used for
1569  * reading/writing page data, they are used for smaller data like reading
1570  * id, status etc
1571  */
1572 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
1573 {
1574         struct nand_chip *chip = mtd_to_nand(mtd);
1575         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1576         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1577         u8 *buf = nandc->data_buffer;
1578         u8 ret = 0x0;
1579
1580         if (host->last_command == NAND_CMD_STATUS) {
1581                 ret = host->status;
1582
1583                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
1584
1585                 return ret;
1586         }
1587
1588         if (nandc->buf_start < nandc->buf_count)
1589                 ret = buf[nandc->buf_start++];
1590
1591         return ret;
1592 }
1593
1594 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
1595 {
1596         struct nand_chip *chip = mtd_to_nand(mtd);
1597         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1598         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1599
1600         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
1601         nandc->buf_start += real_len;
1602 }
1603
1604 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
1605                                  int len)
1606 {
1607         struct nand_chip *chip = mtd_to_nand(mtd);
1608         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1609         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1610
1611         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
1612
1613         nandc->buf_start += real_len;
1614 }
1615
1616 /* we support only one external chip for now */
1617 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
1618 {
1619         struct nand_chip *chip = mtd_to_nand(mtd);
1620         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1621
1622         if (chipnr <= 0)
1623                 return;
1624
1625         dev_warn(nandc->dev, "invalid chip select\n");
1626 }
1627
1628 /*
1629  * NAND controller page layout info
1630  *
1631  * Layout with ECC enabled:
1632  *
1633  * |----------------------|  |---------------------------------|
1634  * |           xx.......yy|  |             *********xx.......yy|
1635  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
1636  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
1637  * |           xx.......yy|  |             *********xx.......yy|
1638  * |----------------------|  |---------------------------------|
1639  *     codeword 1,2..n-1                  codeword n
1640  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
1641  *
1642  * n = Number of codewords in the page
1643  * . = ECC bytes
1644  * * = Spare/free bytes
1645  * x = Unused byte(s)
1646  * y = Reserved byte(s)
1647  *
1648  * 2K page: n = 4, spare = 16 bytes
1649  * 4K page: n = 8, spare = 32 bytes
1650  * 8K page: n = 16, spare = 64 bytes
1651  *
1652  * the qcom nand controller operates at a sub page/codeword level. each
1653  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
1654  * the number of ECC bytes vary based on the ECC strength and the bus width.
1655  *
1656  * the first n - 1 codewords contains 516 bytes of user data, the remaining
1657  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
1658  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
1659  *
1660  * When we access a page with ECC enabled, the reserved bytes(s) are not
1661  * accessible at all. When reading, we fill up these unreadable positions
1662  * with 0xffs. When writing, the controller skips writing the inaccessible
1663  * bytes.
1664  *
1665  * Layout with ECC disabled:
1666  *
1667  * |------------------------------|  |---------------------------------------|
1668  * |         yy          xx.......|  |         bb          *********xx.......|
1669  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
1670  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
1671  * |         yy          xx.......|  |         bb          *********xx.......|
1672  * |------------------------------|  |---------------------------------------|
1673  *         codeword 1,2..n-1                        codeword n
1674  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
1675  *
1676  * n = Number of codewords in the page
1677  * . = ECC bytes
1678  * * = Spare/free bytes
1679  * x = Unused byte(s)
1680  * y = Dummy Bad Bock byte(s)
1681  * b = Real Bad Block byte(s)
1682  * size1/size2 = function of codeword size and 'n'
1683  *
1684  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
1685  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
1686  * Block Markers. In the last codeword, this position contains the real BBM
1687  *
1688  * In order to have a consistent layout between RAW and ECC modes, we assume
1689  * the following OOB layout arrangement:
1690  *
1691  * |-----------|  |--------------------|
1692  * |yyxx.......|  |bb*********xx.......|
1693  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
1694  * |yyxx.......|  |bb*********xx.......|
1695  * |yyxx.......|  |bb*********xx.......|
1696  * |-----------|  |--------------------|
1697  *  first n - 1       nth OOB region
1698  *  OOB regions
1699  *
1700  * n = Number of codewords in the page
1701  * . = ECC bytes
1702  * * = FREE OOB bytes
1703  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
1704  * x = Unused byte(s)
1705  * b = Real bad block byte(s) (inaccessible when ECC enabled)
1706  *
1707  * This layout is read as is when ECC is disabled. When ECC is enabled, the
1708  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
1709  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
1710  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
1711  * the sum of the three).
1712  */
1713 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1714                                    struct mtd_oob_region *oobregion)
1715 {
1716         struct nand_chip *chip = mtd_to_nand(mtd);
1717         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1718         struct nand_ecc_ctrl *ecc = &chip->ecc;
1719
1720         if (section > 1)
1721                 return -ERANGE;
1722
1723         if (!section) {
1724                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
1725                                     host->bbm_size;
1726                 oobregion->offset = 0;
1727         } else {
1728                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
1729                 oobregion->offset = mtd->oobsize - oobregion->length;
1730         }
1731
1732         return 0;
1733 }
1734
1735 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
1736                                      struct mtd_oob_region *oobregion)
1737 {
1738         struct nand_chip *chip = mtd_to_nand(mtd);
1739         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1740         struct nand_ecc_ctrl *ecc = &chip->ecc;
1741
1742         if (section)
1743                 return -ERANGE;
1744
1745         oobregion->length = ecc->steps * 4;
1746         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
1747
1748         return 0;
1749 }
1750
1751 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
1752         .ecc = qcom_nand_ooblayout_ecc,
1753         .free = qcom_nand_ooblayout_free,
1754 };
1755
1756 static int qcom_nand_host_setup(struct qcom_nand_host *host)
1757 {
1758         struct nand_chip *chip = &host->chip;
1759         struct mtd_info *mtd = nand_to_mtd(chip);
1760         struct nand_ecc_ctrl *ecc = &chip->ecc;
1761         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1762         int cwperpage, bad_block_byte;
1763         bool wide_bus;
1764         int ecc_mode = 1;
1765
1766         /*
1767          * the controller requires each step consists of 512 bytes of data.
1768          * bail out if DT has populated a wrong step size.
1769          */
1770         if (ecc->size != NANDC_STEP_SIZE) {
1771                 dev_err(nandc->dev, "invalid ecc size\n");
1772                 return -EINVAL;
1773         }
1774
1775         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
1776
1777         if (ecc->strength >= 8) {
1778                 /* 8 bit ECC defaults to BCH ECC on all platforms */
1779                 host->bch_enabled = true;
1780                 ecc_mode = 1;
1781
1782                 if (wide_bus) {
1783                         host->ecc_bytes_hw = 14;
1784                         host->spare_bytes = 0;
1785                         host->bbm_size = 2;
1786                 } else {
1787                         host->ecc_bytes_hw = 13;
1788                         host->spare_bytes = 2;
1789                         host->bbm_size = 1;
1790                 }
1791         } else {
1792                 /*
1793                  * if the controller supports BCH for 4 bit ECC, the controller
1794                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
1795                  * always 10 bytes
1796                  */
1797                 if (nandc->ecc_modes & ECC_BCH_4BIT) {
1798                         /* BCH */
1799                         host->bch_enabled = true;
1800                         ecc_mode = 0;
1801
1802                         if (wide_bus) {
1803                                 host->ecc_bytes_hw = 8;
1804                                 host->spare_bytes = 2;
1805                                 host->bbm_size = 2;
1806                         } else {
1807                                 host->ecc_bytes_hw = 7;
1808                                 host->spare_bytes = 4;
1809                                 host->bbm_size = 1;
1810                         }
1811                 } else {
1812                         /* RS */
1813                         host->ecc_bytes_hw = 10;
1814
1815                         if (wide_bus) {
1816                                 host->spare_bytes = 0;
1817                                 host->bbm_size = 2;
1818                         } else {
1819                                 host->spare_bytes = 1;
1820                                 host->bbm_size = 1;
1821                         }
1822                 }
1823         }
1824
1825         /*
1826          * we consider ecc->bytes as the sum of all the non-data content in a
1827          * step. It gives us a clean representation of the oob area (even if
1828          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
1829          * ECC and 12 bytes for 4 bit ECC
1830          */
1831         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
1832
1833         ecc->read_page          = qcom_nandc_read_page;
1834         ecc->read_page_raw      = qcom_nandc_read_page_raw;
1835         ecc->read_oob           = qcom_nandc_read_oob;
1836         ecc->write_page         = qcom_nandc_write_page;
1837         ecc->write_page_raw     = qcom_nandc_write_page_raw;
1838         ecc->write_oob          = qcom_nandc_write_oob;
1839
1840         ecc->mode = NAND_ECC_HW;
1841
1842         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
1843
1844         cwperpage = mtd->writesize / ecc->size;
1845
1846         /*
1847          * DATA_UD_BYTES varies based on whether the read/write command protects
1848          * spare data with ECC too. We protect spare data by default, so we set
1849          * it to main + spare data, which are 512 and 4 bytes respectively.
1850          */
1851         host->cw_data = 516;
1852
1853         /*
1854          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
1855          * for 8 bit ECC
1856          */
1857         host->cw_size = host->cw_data + ecc->bytes;
1858
1859         if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
1860                 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
1861                 return -EINVAL;
1862         }
1863
1864         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
1865
1866         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
1867                                 | host->cw_data << UD_SIZE_BYTES
1868                                 | 0 << DISABLE_STATUS_AFTER_WRITE
1869                                 | 5 << NUM_ADDR_CYCLES
1870                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
1871                                 | 0 << STATUS_BFR_READ
1872                                 | 1 << SET_RD_MODE_AFTER_STATUS
1873                                 | host->spare_bytes << SPARE_SIZE_BYTES;
1874
1875         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
1876                                 | 0 <<  CS_ACTIVE_BSY
1877                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
1878                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
1879                                 | 2 << WR_RD_BSY_GAP
1880                                 | wide_bus << WIDE_FLASH
1881                                 | host->bch_enabled << ENABLE_BCH_ECC;
1882
1883         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
1884                                 | host->cw_size << UD_SIZE_BYTES
1885                                 | 5 << NUM_ADDR_CYCLES
1886                                 | 0 << SPARE_SIZE_BYTES;
1887
1888         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
1889                                 | 0 << CS_ACTIVE_BSY
1890                                 | 17 << BAD_BLOCK_BYTE_NUM
1891                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
1892                                 | 2 << WR_RD_BSY_GAP
1893                                 | wide_bus << WIDE_FLASH
1894                                 | 1 << DEV0_CFG1_ECC_DISABLE;
1895
1896         host->ecc_bch_cfg = host->bch_enabled << ECC_CFG_ECC_DISABLE
1897                                 | 0 << ECC_SW_RESET
1898                                 | host->cw_data << ECC_NUM_DATA_BYTES
1899                                 | 1 << ECC_FORCE_CLK_OPEN
1900                                 | ecc_mode << ECC_MODE
1901                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
1902
1903         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
1904
1905         host->clrflashstatus = FS_READY_BSY_N;
1906         host->clrreadstatus = 0xc0;
1907
1908         dev_dbg(nandc->dev,
1909                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
1910                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
1911                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
1912                 cwperpage);
1913
1914         return 0;
1915 }
1916
1917 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
1918 {
1919         int ret;
1920
1921         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
1922         if (ret) {
1923                 dev_err(nandc->dev, "failed to set DMA mask\n");
1924                 return ret;
1925         }
1926
1927         /*
1928          * we use the internal buffer for reading ONFI params, reading small
1929          * data like ID and status, and preforming read-copy-write operations
1930          * when writing to a codeword partially. 532 is the maximum possible
1931          * size of a codeword for our nand controller
1932          */
1933         nandc->buf_size = 532;
1934
1935         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
1936                                         GFP_KERNEL);
1937         if (!nandc->data_buffer)
1938                 return -ENOMEM;
1939
1940         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
1941                                         GFP_KERNEL);
1942         if (!nandc->regs)
1943                 return -ENOMEM;
1944
1945         nandc->reg_read_buf = devm_kzalloc(nandc->dev,
1946                                 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
1947                                 GFP_KERNEL);
1948         if (!nandc->reg_read_buf)
1949                 return -ENOMEM;
1950
1951         nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
1952         if (!nandc->chan) {
1953                 dev_err(nandc->dev, "failed to request slave channel\n");
1954                 return -ENODEV;
1955         }
1956
1957         INIT_LIST_HEAD(&nandc->desc_list);
1958         INIT_LIST_HEAD(&nandc->host_list);
1959
1960         nand_hw_control_init(&nandc->controller);
1961
1962         return 0;
1963 }
1964
1965 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
1966 {
1967         dma_release_channel(nandc->chan);
1968 }
1969
1970 /* one time setup of a few nand controller registers */
1971 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
1972 {
1973         /* kill onenand */
1974         nandc_write(nandc, SFLASHC_BURST_CFG, 0);
1975
1976         /* enable ADM DMA */
1977         nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
1978
1979         /* save the original values of these registers */
1980         nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
1981         nandc->vld = nandc_read(nandc, NAND_DEV_CMD_VLD);
1982
1983         return 0;
1984 }
1985
1986 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
1987                                struct qcom_nand_host *host,
1988                                struct device_node *dn)
1989 {
1990         struct nand_chip *chip = &host->chip;
1991         struct mtd_info *mtd = nand_to_mtd(chip);
1992         struct device *dev = nandc->dev;
1993         int ret;
1994
1995         ret = of_property_read_u32(dn, "reg", &host->cs);
1996         if (ret) {
1997                 dev_err(dev, "can't get chip-select\n");
1998                 return -ENXIO;
1999         }
2000
2001         nand_set_flash_node(chip, dn);
2002         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2003         mtd->owner = THIS_MODULE;
2004         mtd->dev.parent = dev;
2005
2006         chip->cmdfunc           = qcom_nandc_command;
2007         chip->select_chip       = qcom_nandc_select_chip;
2008         chip->read_byte         = qcom_nandc_read_byte;
2009         chip->read_buf          = qcom_nandc_read_buf;
2010         chip->write_buf         = qcom_nandc_write_buf;
2011         chip->onfi_set_features = nand_onfi_get_set_features_notsupp;
2012         chip->onfi_get_features = nand_onfi_get_set_features_notsupp;
2013
2014         /*
2015          * the bad block marker is readable only when we read the last codeword
2016          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2017          * helpers don't allow us to read BB from a nand chip with ECC
2018          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2019          * and block_markbad helpers until we permanently switch to using
2020          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2021          */
2022         chip->block_bad         = qcom_nandc_block_bad;
2023         chip->block_markbad     = qcom_nandc_block_markbad;
2024
2025         chip->controller = &nandc->controller;
2026         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2027                          NAND_SKIP_BBTSCAN;
2028
2029         /* set up initial status value */
2030         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2031
2032         ret = nand_scan_ident(mtd, 1, NULL);
2033         if (ret)
2034                 return ret;
2035
2036         ret = qcom_nand_host_setup(host);
2037         if (ret)
2038                 return ret;
2039
2040         ret = nand_scan_tail(mtd);
2041         if (ret)
2042                 return ret;
2043
2044         return mtd_device_register(mtd, NULL, 0);
2045 }
2046
2047 /* parse custom DT properties here */
2048 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2049 {
2050         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2051         struct device_node *np = nandc->dev->of_node;
2052         int ret;
2053
2054         ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
2055         if (ret) {
2056                 dev_err(nandc->dev, "command CRCI unspecified\n");
2057                 return ret;
2058         }
2059
2060         ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
2061         if (ret) {
2062                 dev_err(nandc->dev, "data CRCI unspecified\n");
2063                 return ret;
2064         }
2065
2066         return 0;
2067 }
2068
2069 static int qcom_nandc_probe(struct platform_device *pdev)
2070 {
2071         struct qcom_nand_controller *nandc;
2072         struct qcom_nand_host *host;
2073         const void *dev_data;
2074         struct device *dev = &pdev->dev;
2075         struct device_node *dn = dev->of_node, *child;
2076         struct resource *res;
2077         int ret;
2078
2079         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2080         if (!nandc)
2081                 return -ENOMEM;
2082
2083         platform_set_drvdata(pdev, nandc);
2084         nandc->dev = dev;
2085
2086         dev_data = of_device_get_match_data(dev);
2087         if (!dev_data) {
2088                 dev_err(&pdev->dev, "failed to get device data\n");
2089                 return -ENODEV;
2090         }
2091
2092         nandc->ecc_modes = (unsigned long)dev_data;
2093
2094         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2095         nandc->base = devm_ioremap_resource(dev, res);
2096         if (IS_ERR(nandc->base))
2097                 return PTR_ERR(nandc->base);
2098
2099         nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2100
2101         nandc->core_clk = devm_clk_get(dev, "core");
2102         if (IS_ERR(nandc->core_clk))
2103                 return PTR_ERR(nandc->core_clk);
2104
2105         nandc->aon_clk = devm_clk_get(dev, "aon");
2106         if (IS_ERR(nandc->aon_clk))
2107                 return PTR_ERR(nandc->aon_clk);
2108
2109         ret = qcom_nandc_parse_dt(pdev);
2110         if (ret)
2111                 return ret;
2112
2113         ret = qcom_nandc_alloc(nandc);
2114         if (ret)
2115                 return ret;
2116
2117         ret = clk_prepare_enable(nandc->core_clk);
2118         if (ret)
2119                 goto err_core_clk;
2120
2121         ret = clk_prepare_enable(nandc->aon_clk);
2122         if (ret)
2123                 goto err_aon_clk;
2124
2125         ret = qcom_nandc_setup(nandc);
2126         if (ret)
2127                 goto err_setup;
2128
2129         for_each_available_child_of_node(dn, child) {
2130                 if (of_device_is_compatible(child, "qcom,nandcs")) {
2131                         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2132                         if (!host) {
2133                                 of_node_put(child);
2134                                 ret = -ENOMEM;
2135                                 goto err_cs_init;
2136                         }
2137
2138                         ret = qcom_nand_host_init(nandc, host, child);
2139                         if (ret) {
2140                                 devm_kfree(dev, host);
2141                                 continue;
2142                         }
2143
2144                         list_add_tail(&host->node, &nandc->host_list);
2145                 }
2146         }
2147
2148         if (list_empty(&nandc->host_list)) {
2149                 ret = -ENODEV;
2150                 goto err_cs_init;
2151         }
2152
2153         return 0;
2154
2155 err_cs_init:
2156         list_for_each_entry(host, &nandc->host_list, node)
2157                 nand_release(nand_to_mtd(&host->chip));
2158 err_setup:
2159         clk_disable_unprepare(nandc->aon_clk);
2160 err_aon_clk:
2161         clk_disable_unprepare(nandc->core_clk);
2162 err_core_clk:
2163         qcom_nandc_unalloc(nandc);
2164
2165         return ret;
2166 }
2167
2168 static int qcom_nandc_remove(struct platform_device *pdev)
2169 {
2170         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2171         struct qcom_nand_host *host;
2172
2173         list_for_each_entry(host, &nandc->host_list, node)
2174                 nand_release(nand_to_mtd(&host->chip));
2175
2176         qcom_nandc_unalloc(nandc);
2177
2178         clk_disable_unprepare(nandc->aon_clk);
2179         clk_disable_unprepare(nandc->core_clk);
2180
2181         return 0;
2182 }
2183
2184 #define EBI2_NANDC_ECC_MODES    (ECC_RS_4BIT | ECC_BCH_8BIT)
2185
2186 /*
2187  * data will hold a struct pointer containing more differences once we support
2188  * more controller variants
2189  */
2190 static const struct of_device_id qcom_nandc_of_match[] = {
2191         {       .compatible = "qcom,ipq806x-nand",
2192                 .data = (void *)EBI2_NANDC_ECC_MODES,
2193         },
2194         {}
2195 };
2196 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2197
2198 static struct platform_driver qcom_nandc_driver = {
2199         .driver = {
2200                 .name = "qcom-nandc",
2201                 .of_match_table = qcom_nandc_of_match,
2202         },
2203         .probe   = qcom_nandc_probe,
2204         .remove  = qcom_nandc_remove,
2205 };
2206 module_platform_driver(qcom_nandc_driver);
2207
2208 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2209 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2210 MODULE_LICENSE("GPL v2");