1 // SPDX-License-Identifier: GPL-2.0-only
4 * This is the generic MTD driver for NAND flash devices. It should be
5 * capable of working with almost all NAND chips currently available.
7 * Additional technical information is available on
8 * http://www.linux-mtd.infradead.org/doc/nand.html
10 * Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com)
11 * 2002-2006 Thomas Gleixner (tglx@linutronix.de)
14 * David Woodhouse for adding multichip support
16 * Aleph One Ltd. and Toby Churchill Ltd. for supporting the
17 * rework for 2K page size chips
20 * Enable cached programming for 2k page size chips
21 * Check, if mtd->ecctype should be set to MTD_ECC_HW
22 * if we have HW ECC support.
23 * BBT table is not serialized, has to be fixed
26 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
28 #include <linux/module.h>
29 #include <linux/delay.h>
30 #include <linux/errno.h>
31 #include <linux/err.h>
32 #include <linux/sched.h>
33 #include <linux/slab.h>
35 #include <linux/types.h>
36 #include <linux/mtd/mtd.h>
37 #include <linux/mtd/nand.h>
38 #include <linux/mtd/nand-ecc-sw-hamming.h>
39 #include <linux/mtd/nand-ecc-sw-bch.h>
40 #include <linux/interrupt.h>
41 #include <linux/bitops.h>
43 #include <linux/mtd/partitions.h>
45 #include <linux/of_gpio.h>
46 #include <linux/gpio/consumer.h>
48 #include "internals.h"
50 static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page,
51 struct mtd_pairing_info *info)
53 int lastpage = (mtd->erasesize / mtd->writesize) - 1;
59 if (!page || (page & 1)) {
61 info->pair = (page + 1) / 2;
64 info->pair = (page + 1 - dist) / 2;
70 static int nand_pairing_dist3_get_wunit(struct mtd_info *mtd,
71 const struct mtd_pairing_info *info)
73 int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2;
74 int page = info->pair * 2;
77 if (!info->group && !info->pair)
80 if (info->pair == lastpair && info->group)
88 if (page >= mtd->erasesize / mtd->writesize)
94 const struct mtd_pairing_scheme dist3_pairing_scheme = {
96 .get_info = nand_pairing_dist3_get_info,
97 .get_wunit = nand_pairing_dist3_get_wunit,
100 static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len)
104 /* Start address must align on block boundary */
105 if (ofs & ((1ULL << chip->phys_erase_shift) - 1)) {
106 pr_debug("%s: unaligned address\n", __func__);
110 /* Length must align on block boundary */
111 if (len & ((1ULL << chip->phys_erase_shift) - 1)) {
112 pr_debug("%s: length not block aligned\n", __func__);
120 * nand_extract_bits - Copy unaligned bits from one buffer to another one
121 * @dst: destination buffer
122 * @dst_off: bit offset at which the writing starts
123 * @src: source buffer
124 * @src_off: bit offset at which the reading starts
125 * @nbits: number of bits to copy from @src to @dst
127 * Copy bits from one memory region to another (overlap authorized).
129 void nand_extract_bits(u8 *dst, unsigned int dst_off, const u8 *src,
130 unsigned int src_off, unsigned int nbits)
140 n = min3(8 - dst_off, 8 - src_off, nbits);
142 tmp = (*src >> src_off) & GENMASK(n - 1, 0);
143 *dst &= ~GENMASK(n - 1 + dst_off, dst_off);
144 *dst |= tmp << dst_off;
161 EXPORT_SYMBOL_GPL(nand_extract_bits);
164 * nand_select_target() - Select a NAND target (A.K.A. die)
165 * @chip: NAND chip object
166 * @cs: the CS line to select. Note that this CS id is always from the chip
167 * PoV, not the controller one
169 * Select a NAND target so that further operations executed on @chip go to the
170 * selected NAND target.
172 void nand_select_target(struct nand_chip *chip, unsigned int cs)
175 * cs should always lie between 0 and nanddev_ntargets(), when that's
176 * not the case it's a bug and the caller should be fixed.
178 if (WARN_ON(cs > nanddev_ntargets(&chip->base)))
183 if (chip->legacy.select_chip)
184 chip->legacy.select_chip(chip, cs);
186 EXPORT_SYMBOL_GPL(nand_select_target);
189 * nand_deselect_target() - Deselect the currently selected target
190 * @chip: NAND chip object
192 * Deselect the currently selected NAND target. The result of operations
193 * executed on @chip after the target has been deselected is undefined.
195 void nand_deselect_target(struct nand_chip *chip)
197 if (chip->legacy.select_chip)
198 chip->legacy.select_chip(chip, -1);
202 EXPORT_SYMBOL_GPL(nand_deselect_target);
205 * nand_release_device - [GENERIC] release chip
206 * @chip: NAND chip object
208 * Release chip lock and wake up anyone waiting on the device.
210 static void nand_release_device(struct nand_chip *chip)
212 /* Release the controller and the chip */
213 mutex_unlock(&chip->controller->lock);
214 mutex_unlock(&chip->lock);
218 * nand_bbm_get_next_page - Get the next page for bad block markers
219 * @chip: NAND chip object
220 * @page: First page to start checking for bad block marker usage
222 * Returns an integer that corresponds to the page offset within a block, for
223 * a page that is used to store bad block markers. If no more pages are
224 * available, -EINVAL is returned.
226 int nand_bbm_get_next_page(struct nand_chip *chip, int page)
228 struct mtd_info *mtd = nand_to_mtd(chip);
229 int last_page = ((mtd->erasesize - mtd->writesize) >>
230 chip->page_shift) & chip->pagemask;
231 unsigned int bbm_flags = NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE
234 if (page == 0 && !(chip->options & bbm_flags))
236 if (page == 0 && chip->options & NAND_BBM_FIRSTPAGE)
238 if (page <= 1 && chip->options & NAND_BBM_SECONDPAGE)
240 if (page <= last_page && chip->options & NAND_BBM_LASTPAGE)
247 * nand_block_bad - [DEFAULT] Read bad block marker from the chip
248 * @chip: NAND chip object
249 * @ofs: offset from device start
251 * Check, if the block is bad.
253 static int nand_block_bad(struct nand_chip *chip, loff_t ofs)
255 int first_page, page_offset;
259 first_page = (int)(ofs >> chip->page_shift) & chip->pagemask;
260 page_offset = nand_bbm_get_next_page(chip, 0);
262 while (page_offset >= 0) {
263 res = chip->ecc.read_oob(chip, first_page + page_offset);
267 bad = chip->oob_poi[chip->badblockpos];
269 if (likely(chip->badblockbits == 8))
272 res = hweight8(bad) < chip->badblockbits;
276 page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
283 * nand_region_is_secured() - Check if the region is secured
284 * @chip: NAND chip object
285 * @offset: Offset of the region to check
286 * @size: Size of the region to check
288 * Checks if the region is secured by comparing the offset and size with the
289 * list of secure regions obtained from DT. Returns true if the region is
290 * secured else false.
292 static bool nand_region_is_secured(struct nand_chip *chip, loff_t offset, u64 size)
296 /* Skip touching the secure regions if present */
297 for (i = 0; i < chip->nr_secure_regions; i++) {
298 const struct nand_secure_region *region = &chip->secure_regions[i];
300 if (offset + size <= region->offset ||
301 offset >= region->offset + region->size)
304 pr_debug("%s: Region 0x%llx - 0x%llx is secured!",
305 __func__, offset, offset + size);
313 static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs)
315 struct mtd_info *mtd = nand_to_mtd(chip);
317 if (chip->options & NAND_NO_BBM_QUIRK)
320 /* Check if the region is secured */
321 if (nand_region_is_secured(chip, ofs, mtd->erasesize))
324 if (mtd_check_expert_analysis_mode())
327 if (chip->legacy.block_bad)
328 return chip->legacy.block_bad(chip, ofs);
330 return nand_block_bad(chip, ofs);
334 * nand_get_device - [GENERIC] Get chip for selected access
335 * @chip: NAND chip structure
337 * Lock the device and its controller for exclusive access
339 static void nand_get_device(struct nand_chip *chip)
341 /* Wait until the device is resumed. */
343 mutex_lock(&chip->lock);
344 if (!chip->suspended) {
345 mutex_lock(&chip->controller->lock);
348 mutex_unlock(&chip->lock);
350 wait_event(chip->resume_wq, !chip->suspended);
355 * nand_check_wp - [GENERIC] check if the chip is write protected
356 * @chip: NAND chip object
358 * Check, if the device is write protected. The function expects, that the
359 * device is already selected.
361 static int nand_check_wp(struct nand_chip *chip)
366 /* Broken xD cards report WP despite being writable */
367 if (chip->options & NAND_BROKEN_XD)
370 /* Check the WP bit */
371 ret = nand_status_op(chip, &status);
375 return status & NAND_STATUS_WP ? 0 : 1;
379 * nand_fill_oob - [INTERN] Transfer client buffer to oob
380 * @chip: NAND chip object
381 * @oob: oob data buffer
382 * @len: oob data write length
383 * @ops: oob ops structure
385 static uint8_t *nand_fill_oob(struct nand_chip *chip, uint8_t *oob, size_t len,
386 struct mtd_oob_ops *ops)
388 struct mtd_info *mtd = nand_to_mtd(chip);
392 * Initialise to all 0xFF, to avoid the possibility of left over OOB
393 * data from a previous OOB read.
395 memset(chip->oob_poi, 0xff, mtd->oobsize);
399 case MTD_OPS_PLACE_OOB:
401 memcpy(chip->oob_poi + ops->ooboffs, oob, len);
404 case MTD_OPS_AUTO_OOB:
405 ret = mtd_ooblayout_set_databytes(mtd, oob, chip->oob_poi,
417 * nand_do_write_oob - [MTD Interface] NAND write out-of-band
418 * @chip: NAND chip object
419 * @to: offset to write to
420 * @ops: oob operation description structure
422 * NAND write out-of-band.
424 static int nand_do_write_oob(struct nand_chip *chip, loff_t to,
425 struct mtd_oob_ops *ops)
427 struct mtd_info *mtd = nand_to_mtd(chip);
428 int chipnr, page, status, len, ret;
430 pr_debug("%s: to = 0x%08x, len = %i\n",
431 __func__, (unsigned int)to, (int)ops->ooblen);
433 len = mtd_oobavail(mtd, ops);
435 /* Do not allow write past end of page */
436 if ((ops->ooboffs + ops->ooblen) > len) {
437 pr_debug("%s: attempt to write past end of page\n",
442 /* Check if the region is secured */
443 if (nand_region_is_secured(chip, to, ops->ooblen))
446 chipnr = (int)(to >> chip->chip_shift);
449 * Reset the chip. Some chips (like the Toshiba TC5832DC found in one
450 * of my DiskOnChip 2000 test units) will clear the whole data page too
451 * if we don't do this. I have no clue why, but I seem to have 'fixed'
452 * it in the doc2000 driver in August 1999. dwmw2.
454 ret = nand_reset(chip, chipnr);
458 nand_select_target(chip, chipnr);
460 /* Shift to get page */
461 page = (int)(to >> chip->page_shift);
463 /* Check, if it is write protected */
464 if (nand_check_wp(chip)) {
465 nand_deselect_target(chip);
469 /* Invalidate the page cache, if we write to the cached page */
470 if (page == chip->pagecache.page)
471 chip->pagecache.page = -1;
473 nand_fill_oob(chip, ops->oobbuf, ops->ooblen, ops);
475 if (ops->mode == MTD_OPS_RAW)
476 status = chip->ecc.write_oob_raw(chip, page & chip->pagemask);
478 status = chip->ecc.write_oob(chip, page & chip->pagemask);
480 nand_deselect_target(chip);
485 ops->oobretlen = ops->ooblen;
491 * nand_default_block_markbad - [DEFAULT] mark a block bad via bad block marker
492 * @chip: NAND chip object
493 * @ofs: offset from device start
495 * This is the default implementation, which can be overridden by a hardware
496 * specific driver. It provides the details for writing a bad block marker to a
499 static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs)
501 struct mtd_info *mtd = nand_to_mtd(chip);
502 struct mtd_oob_ops ops;
503 uint8_t buf[2] = { 0, 0 };
504 int ret = 0, res, page_offset;
506 memset(&ops, 0, sizeof(ops));
508 ops.ooboffs = chip->badblockpos;
509 if (chip->options & NAND_BUSWIDTH_16) {
510 ops.ooboffs &= ~0x01;
511 ops.len = ops.ooblen = 2;
513 ops.len = ops.ooblen = 1;
515 ops.mode = MTD_OPS_PLACE_OOB;
517 page_offset = nand_bbm_get_next_page(chip, 0);
519 while (page_offset >= 0) {
520 res = nand_do_write_oob(chip,
521 ofs + (page_offset * mtd->writesize),
527 page_offset = nand_bbm_get_next_page(chip, page_offset + 1);
534 * nand_markbad_bbm - mark a block by updating the BBM
535 * @chip: NAND chip object
536 * @ofs: offset of the block to mark bad
538 int nand_markbad_bbm(struct nand_chip *chip, loff_t ofs)
540 if (chip->legacy.block_markbad)
541 return chip->legacy.block_markbad(chip, ofs);
543 return nand_default_block_markbad(chip, ofs);
547 * nand_block_markbad_lowlevel - mark a block bad
548 * @chip: NAND chip object
549 * @ofs: offset from device start
551 * This function performs the generic NAND bad block marking steps (i.e., bad
552 * block table(s) and/or marker(s)). We only allow the hardware driver to
553 * specify how to write bad block markers to OOB (chip->legacy.block_markbad).
555 * We try operations in the following order:
557 * (1) erase the affected block, to allow OOB marker to be written cleanly
558 * (2) write bad block marker to OOB area of affected block (unless flag
559 * NAND_BBT_NO_OOB_BBM is present)
562 * Note that we retain the first error encountered in (2) or (3), finish the
563 * procedures, and dump the error in the end.
565 static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs)
567 struct mtd_info *mtd = nand_to_mtd(chip);
570 if (!(chip->bbt_options & NAND_BBT_NO_OOB_BBM)) {
571 struct erase_info einfo;
573 /* Attempt erase before marking OOB */
574 memset(&einfo, 0, sizeof(einfo));
576 einfo.len = 1ULL << chip->phys_erase_shift;
577 nand_erase_nand(chip, &einfo, 0);
579 /* Write bad block marker to OOB */
580 nand_get_device(chip);
582 ret = nand_markbad_bbm(chip, ofs);
583 nand_release_device(chip);
586 /* Mark block bad in BBT */
588 res = nand_markbad_bbt(chip, ofs);
594 mtd->ecc_stats.badblocks++;
600 * nand_block_isreserved - [GENERIC] Check if a block is marked reserved.
601 * @mtd: MTD device structure
602 * @ofs: offset from device start
604 * Check if the block is marked as reserved.
606 static int nand_block_isreserved(struct mtd_info *mtd, loff_t ofs)
608 struct nand_chip *chip = mtd_to_nand(mtd);
612 /* Return info from the table */
613 return nand_isreserved_bbt(chip, ofs);
617 * nand_block_checkbad - [GENERIC] Check if a block is marked bad
618 * @chip: NAND chip object
619 * @ofs: offset from device start
620 * @allowbbt: 1, if its allowed to access the bbt area
622 * Check, if the block is bad. Either by reading the bad block table or
623 * calling of the scan function.
625 static int nand_block_checkbad(struct nand_chip *chip, loff_t ofs, int allowbbt)
627 /* Return info from the table */
629 return nand_isbad_bbt(chip, ofs, allowbbt);
631 return nand_isbad_bbm(chip, ofs);
635 * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1
636 * @chip: NAND chip structure
637 * @timeout_ms: Timeout in ms
639 * Poll the STATUS register using ->exec_op() until the RDY bit becomes 1.
640 * If that does not happen whitin the specified timeout, -ETIMEDOUT is
643 * This helper is intended to be used when the controller does not have access
644 * to the NAND R/B pin.
646 * Be aware that calling this helper from an ->exec_op() implementation means
647 * ->exec_op() must be re-entrant.
649 * Return 0 if the NAND chip is ready, a negative error otherwise.
651 int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
653 const struct nand_interface_config *conf;
657 if (!nand_has_exec_op(chip))
660 /* Wait tWB before polling the STATUS reg. */
661 conf = nand_get_interface_config(chip);
662 ndelay(NAND_COMMON_TIMING_NS(conf, tWB_max));
664 ret = nand_status_op(chip, NULL);
669 * +1 below is necessary because if we are now in the last fraction
670 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
671 * small jiffy fraction - possibly leading to false timeout
673 timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
675 ret = nand_read_data_op(chip, &status, sizeof(status), true,
680 if (status & NAND_STATUS_READY)
684 * Typical lowest execution time for a tR on most NANDs is 10us,
685 * use this as polling delay before doing something smarter (ie.
686 * deriving a delay from the timeout value, timeout_ms/ratio).
689 } while (time_before(jiffies, timeout_ms));
692 * We have to exit READ_STATUS mode in order to read real data on the
693 * bus in case the WAITRDY instruction is preceding a DATA_IN
696 nand_exit_status_op(chip);
701 return status & NAND_STATUS_READY ? 0 : -ETIMEDOUT;
703 EXPORT_SYMBOL_GPL(nand_soft_waitrdy);
706 * nand_gpio_waitrdy - Poll R/B GPIO pin until ready
707 * @chip: NAND chip structure
708 * @gpiod: GPIO descriptor of R/B pin
709 * @timeout_ms: Timeout in ms
711 * Poll the R/B GPIO pin until it becomes ready. If that does not happen
712 * whitin the specified timeout, -ETIMEDOUT is returned.
714 * This helper is intended to be used when the controller has access to the
715 * NAND R/B pin over GPIO.
717 * Return 0 if the R/B pin indicates chip is ready, a negative error otherwise.
719 int nand_gpio_waitrdy(struct nand_chip *chip, struct gpio_desc *gpiod,
720 unsigned long timeout_ms)
724 * Wait until R/B pin indicates chip is ready or timeout occurs.
725 * +1 below is necessary because if we are now in the last fraction
726 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
727 * small jiffy fraction - possibly leading to false timeout.
729 timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
731 if (gpiod_get_value_cansleep(gpiod))
735 } while (time_before(jiffies, timeout_ms));
737 return gpiod_get_value_cansleep(gpiod) ? 0 : -ETIMEDOUT;
739 EXPORT_SYMBOL_GPL(nand_gpio_waitrdy);
742 * panic_nand_wait - [GENERIC] wait until the command is done
743 * @chip: NAND chip structure
746 * Wait for command done. This is a helper function for nand_wait used when
747 * we are in interrupt context. May happen when in panic and trying to write
748 * an oops through mtdoops.
750 void panic_nand_wait(struct nand_chip *chip, unsigned long timeo)
753 for (i = 0; i < timeo; i++) {
754 if (chip->legacy.dev_ready) {
755 if (chip->legacy.dev_ready(chip))
761 ret = nand_read_data_op(chip, &status, sizeof(status),
766 if (status & NAND_STATUS_READY)
773 static bool nand_supports_get_features(struct nand_chip *chip, int addr)
775 return (chip->parameters.supports_set_get_features &&
776 test_bit(addr, chip->parameters.get_feature_list));
779 static bool nand_supports_set_features(struct nand_chip *chip, int addr)
781 return (chip->parameters.supports_set_get_features &&
782 test_bit(addr, chip->parameters.set_feature_list));
786 * nand_reset_interface - Reset data interface and timings
787 * @chip: The NAND chip
788 * @chipnr: Internal die id
790 * Reset the Data interface and timings to ONFI mode 0.
792 * Returns 0 for success or negative error code otherwise.
794 static int nand_reset_interface(struct nand_chip *chip, int chipnr)
796 const struct nand_controller_ops *ops = chip->controller->ops;
799 if (!nand_controller_can_setup_interface(chip))
803 * The ONFI specification says:
805 * To transition from NV-DDR or NV-DDR2 to the SDR data
806 * interface, the host shall use the Reset (FFh) command
807 * using SDR timing mode 0. A device in any timing mode is
808 * required to recognize Reset (FFh) command issued in SDR
812 * Configure the data interface in SDR mode and set the
813 * timings to timing mode 0.
816 chip->current_interface_config = nand_get_reset_interface_config();
817 ret = ops->setup_interface(chip, chipnr,
818 chip->current_interface_config);
820 pr_err("Failed to configure data interface to SDR timing mode 0\n");
826 * nand_setup_interface - Setup the best data interface and timings
827 * @chip: The NAND chip
828 * @chipnr: Internal die id
830 * Configure what has been reported to be the best data interface and NAND
831 * timings supported by the chip and the driver.
833 * Returns 0 for success or negative error code otherwise.
835 static int nand_setup_interface(struct nand_chip *chip, int chipnr)
837 const struct nand_controller_ops *ops = chip->controller->ops;
838 u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = { }, request;
841 if (!nand_controller_can_setup_interface(chip))
845 * A nand_reset_interface() put both the NAND chip and the NAND
846 * controller in timings mode 0. If the default mode for this chip is
847 * also 0, no need to proceed to the change again. Plus, at probe time,
848 * nand_setup_interface() uses ->set/get_features() which would
849 * fail anyway as the parameter page is not available yet.
851 if (!chip->best_interface_config)
854 request = chip->best_interface_config->timings.mode;
855 if (nand_interface_is_sdr(chip->best_interface_config))
856 request |= ONFI_DATA_INTERFACE_SDR;
858 request |= ONFI_DATA_INTERFACE_NVDDR;
859 tmode_param[0] = request;
861 /* Change the mode on the chip side (if supported by the NAND chip) */
862 if (nand_supports_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) {
863 nand_select_target(chip, chipnr);
864 ret = nand_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
866 nand_deselect_target(chip);
871 /* Change the mode on the controller side */
872 ret = ops->setup_interface(chip, chipnr, chip->best_interface_config);
876 /* Check the mode has been accepted by the chip, if supported */
877 if (!nand_supports_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE))
878 goto update_interface_config;
880 memset(tmode_param, 0, ONFI_SUBFEATURE_PARAM_LEN);
881 nand_select_target(chip, chipnr);
882 ret = nand_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE,
884 nand_deselect_target(chip);
888 if (request != tmode_param[0]) {
889 pr_warn("%s timing mode %d not acknowledged by the NAND chip\n",
890 nand_interface_is_nvddr(chip->best_interface_config) ? "NV-DDR" : "SDR",
891 chip->best_interface_config->timings.mode);
892 pr_debug("NAND chip would work in %s timing mode %d\n",
893 tmode_param[0] & ONFI_DATA_INTERFACE_NVDDR ? "NV-DDR" : "SDR",
894 (unsigned int)ONFI_TIMING_MODE_PARAM(tmode_param[0]));
898 update_interface_config:
899 chip->current_interface_config = chip->best_interface_config;
905 * Fallback to mode 0 if the chip explicitly did not ack the chosen
908 nand_reset_interface(chip, chipnr);
909 nand_select_target(chip, chipnr);
911 nand_deselect_target(chip);
917 * nand_choose_best_sdr_timings - Pick up the best SDR timings that both the
918 * NAND controller and the NAND chip support
919 * @chip: the NAND chip
920 * @iface: the interface configuration (can eventually be updated)
921 * @spec_timings: specific timings, when not fitting the ONFI specification
923 * If specific timings are provided, use them. Otherwise, retrieve supported
924 * timing modes from ONFI information.
926 int nand_choose_best_sdr_timings(struct nand_chip *chip,
927 struct nand_interface_config *iface,
928 struct nand_sdr_timings *spec_timings)
930 const struct nand_controller_ops *ops = chip->controller->ops;
931 int best_mode = 0, mode, ret = -EOPNOTSUPP;
933 iface->type = NAND_SDR_IFACE;
936 iface->timings.sdr = *spec_timings;
937 iface->timings.mode = onfi_find_closest_sdr_mode(spec_timings);
939 /* Verify the controller supports the requested interface */
940 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
943 chip->best_interface_config = iface;
947 /* Fallback to slower modes */
948 best_mode = iface->timings.mode;
949 } else if (chip->parameters.onfi) {
950 best_mode = fls(chip->parameters.onfi->sdr_timing_modes) - 1;
953 for (mode = best_mode; mode >= 0; mode--) {
954 onfi_fill_interface_config(chip, iface, NAND_SDR_IFACE, mode);
956 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
959 chip->best_interface_config = iface;
968 * nand_choose_best_nvddr_timings - Pick up the best NVDDR timings that both the
969 * NAND controller and the NAND chip support
970 * @chip: the NAND chip
971 * @iface: the interface configuration (can eventually be updated)
972 * @spec_timings: specific timings, when not fitting the ONFI specification
974 * If specific timings are provided, use them. Otherwise, retrieve supported
975 * timing modes from ONFI information.
977 int nand_choose_best_nvddr_timings(struct nand_chip *chip,
978 struct nand_interface_config *iface,
979 struct nand_nvddr_timings *spec_timings)
981 const struct nand_controller_ops *ops = chip->controller->ops;
982 int best_mode = 0, mode, ret = -EOPNOTSUPP;
984 iface->type = NAND_NVDDR_IFACE;
987 iface->timings.nvddr = *spec_timings;
988 iface->timings.mode = onfi_find_closest_nvddr_mode(spec_timings);
990 /* Verify the controller supports the requested interface */
991 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
994 chip->best_interface_config = iface;
998 /* Fallback to slower modes */
999 best_mode = iface->timings.mode;
1000 } else if (chip->parameters.onfi) {
1001 best_mode = fls(chip->parameters.onfi->nvddr_timing_modes) - 1;
1004 for (mode = best_mode; mode >= 0; mode--) {
1005 onfi_fill_interface_config(chip, iface, NAND_NVDDR_IFACE, mode);
1007 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
1010 chip->best_interface_config = iface;
1019 * nand_choose_best_timings - Pick up the best NVDDR or SDR timings that both
1020 * NAND controller and the NAND chip support
1021 * @chip: the NAND chip
1022 * @iface: the interface configuration (can eventually be updated)
1024 * If specific timings are provided, use them. Otherwise, retrieve supported
1025 * timing modes from ONFI information.
1027 static int nand_choose_best_timings(struct nand_chip *chip,
1028 struct nand_interface_config *iface)
1032 /* Try the fastest timings: NV-DDR */
1033 ret = nand_choose_best_nvddr_timings(chip, iface, NULL);
1037 /* Fallback to SDR timings otherwise */
1038 return nand_choose_best_sdr_timings(chip, iface, NULL);
1042 * nand_choose_interface_config - find the best data interface and timings
1043 * @chip: The NAND chip
1045 * Find the best data interface and NAND timings supported by the chip
1046 * and the driver. Eventually let the NAND manufacturer driver propose his own
1049 * After this function nand_chip->interface_config is initialized with the best
1050 * timing mode available.
1052 * Returns 0 for success or negative error code otherwise.
1054 static int nand_choose_interface_config(struct nand_chip *chip)
1056 struct nand_interface_config *iface;
1059 if (!nand_controller_can_setup_interface(chip))
1062 iface = kzalloc(sizeof(*iface), GFP_KERNEL);
1066 if (chip->ops.choose_interface_config)
1067 ret = chip->ops.choose_interface_config(chip, iface);
1069 ret = nand_choose_best_timings(chip, iface);
1078 * nand_fill_column_cycles - fill the column cycles of an address
1079 * @chip: The NAND chip
1080 * @addrs: Array of address cycles to fill
1081 * @offset_in_page: The offset in the page
1083 * Fills the first or the first two bytes of the @addrs field depending
1084 * on the NAND bus width and the page size.
1086 * Returns the number of cycles needed to encode the column, or a negative
1087 * error code in case one of the arguments is invalid.
1089 static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs,
1090 unsigned int offset_in_page)
1092 struct mtd_info *mtd = nand_to_mtd(chip);
1094 /* Make sure the offset is less than the actual page size. */
1095 if (offset_in_page > mtd->writesize + mtd->oobsize)
1099 * On small page NANDs, there's a dedicated command to access the OOB
1100 * area, and the column address is relative to the start of the OOB
1101 * area, not the start of the page. Asjust the address accordingly.
1103 if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize)
1104 offset_in_page -= mtd->writesize;
1107 * The offset in page is expressed in bytes, if the NAND bus is 16-bit
1108 * wide, then it must be divided by 2.
1110 if (chip->options & NAND_BUSWIDTH_16) {
1111 if (WARN_ON(offset_in_page % 2))
1114 offset_in_page /= 2;
1117 addrs[0] = offset_in_page;
1120 * Small page NANDs use 1 cycle for the columns, while large page NANDs
1123 if (mtd->writesize <= 512)
1126 addrs[1] = offset_in_page >> 8;
1131 static int nand_sp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1132 unsigned int offset_in_page, void *buf,
1135 const struct nand_interface_config *conf =
1136 nand_get_interface_config(chip);
1137 struct mtd_info *mtd = nand_to_mtd(chip);
1139 struct nand_op_instr instrs[] = {
1140 NAND_OP_CMD(NAND_CMD_READ0, 0),
1141 NAND_OP_ADDR(3, addrs, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1142 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1143 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1144 NAND_OP_DATA_IN(len, buf, 0),
1146 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1149 /* Drop the DATA_IN instruction if len is set to 0. */
1153 if (offset_in_page >= mtd->writesize)
1154 instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1155 else if (offset_in_page >= 256 &&
1156 !(chip->options & NAND_BUSWIDTH_16))
1157 instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1159 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1164 addrs[2] = page >> 8;
1166 if (chip->options & NAND_ROW_ADDR_3) {
1167 addrs[3] = page >> 16;
1168 instrs[1].ctx.addr.naddrs++;
1171 return nand_exec_op(chip, &op);
1174 static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
1175 unsigned int offset_in_page, void *buf,
1178 const struct nand_interface_config *conf =
1179 nand_get_interface_config(chip);
1181 struct nand_op_instr instrs[] = {
1182 NAND_OP_CMD(NAND_CMD_READ0, 0),
1183 NAND_OP_ADDR(4, addrs, 0),
1184 NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1185 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1186 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1187 NAND_OP_DATA_IN(len, buf, 0),
1189 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1192 /* Drop the DATA_IN instruction if len is set to 0. */
1196 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1201 addrs[3] = page >> 8;
1203 if (chip->options & NAND_ROW_ADDR_3) {
1204 addrs[4] = page >> 16;
1205 instrs[1].ctx.addr.naddrs++;
1208 return nand_exec_op(chip, &op);
1211 static void rawnand_cap_cont_reads(struct nand_chip *chip)
1213 struct nand_memory_organization *memorg;
1214 unsigned int pages_per_lun, first_lun, last_lun;
1216 memorg = nanddev_get_memorg(&chip->base);
1217 pages_per_lun = memorg->pages_per_eraseblock * memorg->eraseblocks_per_lun;
1218 first_lun = chip->cont_read.first_page / pages_per_lun;
1219 last_lun = chip->cont_read.last_page / pages_per_lun;
1221 /* Prevent sequential cache reads across LUN boundaries */
1222 if (first_lun != last_lun)
1223 chip->cont_read.pause_page = first_lun * pages_per_lun + pages_per_lun - 1;
1225 chip->cont_read.pause_page = chip->cont_read.last_page;
1228 static int nand_lp_exec_cont_read_page_op(struct nand_chip *chip, unsigned int page,
1229 unsigned int offset_in_page, void *buf,
1230 unsigned int len, bool check_only)
1232 const struct nand_interface_config *conf =
1233 nand_get_interface_config(chip);
1235 struct nand_op_instr start_instrs[] = {
1236 NAND_OP_CMD(NAND_CMD_READ0, 0),
1237 NAND_OP_ADDR(4, addrs, 0),
1238 NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1239 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), 0),
1240 NAND_OP_CMD(NAND_CMD_READCACHESEQ, NAND_COMMON_TIMING_NS(conf, tWB_max)),
1241 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1242 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1243 NAND_OP_DATA_IN(len, buf, 0),
1245 struct nand_op_instr cont_instrs[] = {
1246 NAND_OP_CMD(page == chip->cont_read.pause_page ?
1247 NAND_CMD_READCACHEEND : NAND_CMD_READCACHESEQ,
1248 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1249 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1250 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1251 NAND_OP_DATA_IN(len, buf, 0),
1253 struct nand_operation start_op = NAND_OPERATION(chip->cur_cs, start_instrs);
1254 struct nand_operation cont_op = NAND_OPERATION(chip->cur_cs, cont_instrs);
1262 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1267 addrs[3] = page >> 8;
1269 if (chip->options & NAND_ROW_ADDR_3) {
1270 addrs[4] = page >> 16;
1271 start_instrs[1].ctx.addr.naddrs++;
1274 /* Check if cache reads are supported */
1276 if (nand_check_op(chip, &start_op) || nand_check_op(chip, &cont_op))
1282 if (page == chip->cont_read.first_page)
1283 ret = nand_exec_op(chip, &start_op);
1285 ret = nand_exec_op(chip, &cont_op);
1289 if (!chip->cont_read.ongoing)
1292 if (page == chip->cont_read.pause_page &&
1293 page != chip->cont_read.last_page) {
1294 chip->cont_read.first_page = chip->cont_read.pause_page + 1;
1295 rawnand_cap_cont_reads(chip);
1296 } else if (page == chip->cont_read.last_page) {
1297 chip->cont_read.ongoing = false;
1303 static bool rawnand_cont_read_ongoing(struct nand_chip *chip, unsigned int page)
1305 return chip->cont_read.ongoing && page >= chip->cont_read.first_page;
1309 * nand_read_page_op - Do a READ PAGE operation
1310 * @chip: The NAND chip
1311 * @page: page to read
1312 * @offset_in_page: offset within the page
1313 * @buf: buffer used to store the data
1314 * @len: length of the buffer
1316 * This function issues a READ PAGE operation.
1317 * This function does not select/unselect the CS line.
1319 * Returns 0 on success, a negative error code otherwise.
1321 int nand_read_page_op(struct nand_chip *chip, unsigned int page,
1322 unsigned int offset_in_page, void *buf, unsigned int len)
1324 struct mtd_info *mtd = nand_to_mtd(chip);
1329 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1332 if (nand_has_exec_op(chip)) {
1333 if (mtd->writesize > 512) {
1334 if (rawnand_cont_read_ongoing(chip, page))
1335 return nand_lp_exec_cont_read_page_op(chip, page,
1339 return nand_lp_exec_read_page_op(chip, page,
1340 offset_in_page, buf,
1344 return nand_sp_exec_read_page_op(chip, page, offset_in_page,
1348 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, offset_in_page, page);
1350 chip->legacy.read_buf(chip, buf, len);
1354 EXPORT_SYMBOL_GPL(nand_read_page_op);
1357 * nand_read_param_page_op - Do a READ PARAMETER PAGE operation
1358 * @chip: The NAND chip
1359 * @page: parameter page to read
1360 * @buf: buffer used to store the data
1361 * @len: length of the buffer
1363 * This function issues a READ PARAMETER PAGE operation.
1364 * This function does not select/unselect the CS line.
1366 * Returns 0 on success, a negative error code otherwise.
1368 int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf,
1377 if (nand_has_exec_op(chip)) {
1378 const struct nand_interface_config *conf =
1379 nand_get_interface_config(chip);
1380 struct nand_op_instr instrs[] = {
1381 NAND_OP_CMD(NAND_CMD_PARAM, 0),
1382 NAND_OP_ADDR(1, &page,
1383 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1384 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max),
1385 NAND_COMMON_TIMING_NS(conf, tRR_min)),
1386 NAND_OP_8BIT_DATA_IN(len, buf, 0),
1388 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1390 /* Drop the DATA_IN instruction if len is set to 0. */
1394 return nand_exec_op(chip, &op);
1397 chip->legacy.cmdfunc(chip, NAND_CMD_PARAM, page, -1);
1398 for (i = 0; i < len; i++)
1399 p[i] = chip->legacy.read_byte(chip);
1405 * nand_change_read_column_op - Do a CHANGE READ COLUMN operation
1406 * @chip: The NAND chip
1407 * @offset_in_page: offset within the page
1408 * @buf: buffer used to store the data
1409 * @len: length of the buffer
1410 * @force_8bit: force 8-bit bus access
1412 * This function issues a CHANGE READ COLUMN operation.
1413 * This function does not select/unselect the CS line.
1415 * Returns 0 on success, a negative error code otherwise.
1417 int nand_change_read_column_op(struct nand_chip *chip,
1418 unsigned int offset_in_page, void *buf,
1419 unsigned int len, bool force_8bit)
1421 struct mtd_info *mtd = nand_to_mtd(chip);
1426 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1429 /* Small page NANDs do not support column change. */
1430 if (mtd->writesize <= 512)
1433 if (nand_has_exec_op(chip)) {
1434 const struct nand_interface_config *conf =
1435 nand_get_interface_config(chip);
1437 struct nand_op_instr instrs[] = {
1438 NAND_OP_CMD(NAND_CMD_RNDOUT, 0),
1439 NAND_OP_ADDR(2, addrs, 0),
1440 NAND_OP_CMD(NAND_CMD_RNDOUTSTART,
1441 NAND_COMMON_TIMING_NS(conf, tCCS_min)),
1442 NAND_OP_DATA_IN(len, buf, 0),
1444 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1447 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1451 /* Drop the DATA_IN instruction if len is set to 0. */
1455 instrs[3].ctx.data.force_8bit = force_8bit;
1457 return nand_exec_op(chip, &op);
1460 chip->legacy.cmdfunc(chip, NAND_CMD_RNDOUT, offset_in_page, -1);
1462 chip->legacy.read_buf(chip, buf, len);
1466 EXPORT_SYMBOL_GPL(nand_change_read_column_op);
1469 * nand_read_oob_op - Do a READ OOB operation
1470 * @chip: The NAND chip
1471 * @page: page to read
1472 * @offset_in_oob: offset within the OOB area
1473 * @buf: buffer used to store the data
1474 * @len: length of the buffer
1476 * This function issues a READ OOB operation.
1477 * This function does not select/unselect the CS line.
1479 * Returns 0 on success, a negative error code otherwise.
1481 int nand_read_oob_op(struct nand_chip *chip, unsigned int page,
1482 unsigned int offset_in_oob, void *buf, unsigned int len)
1484 struct mtd_info *mtd = nand_to_mtd(chip);
1489 if (offset_in_oob + len > mtd->oobsize)
1492 if (nand_has_exec_op(chip))
1493 return nand_read_page_op(chip, page,
1494 mtd->writesize + offset_in_oob,
1497 chip->legacy.cmdfunc(chip, NAND_CMD_READOOB, offset_in_oob, page);
1499 chip->legacy.read_buf(chip, buf, len);
1503 EXPORT_SYMBOL_GPL(nand_read_oob_op);
1505 static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page,
1506 unsigned int offset_in_page, const void *buf,
1507 unsigned int len, bool prog)
1509 const struct nand_interface_config *conf =
1510 nand_get_interface_config(chip);
1511 struct mtd_info *mtd = nand_to_mtd(chip);
1513 struct nand_op_instr instrs[] = {
1515 * The first instruction will be dropped if we're dealing
1516 * with a large page NAND and adjusted if we're dealing
1517 * with a small page NAND and the page offset is > 255.
1519 NAND_OP_CMD(NAND_CMD_READ0, 0),
1520 NAND_OP_CMD(NAND_CMD_SEQIN, 0),
1521 NAND_OP_ADDR(0, addrs, NAND_COMMON_TIMING_NS(conf, tADL_min)),
1522 NAND_OP_DATA_OUT(len, buf, 0),
1523 NAND_OP_CMD(NAND_CMD_PAGEPROG,
1524 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1525 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max), 0),
1527 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1528 int naddrs = nand_fill_column_cycles(chip, addrs, offset_in_page);
1533 addrs[naddrs++] = page;
1534 addrs[naddrs++] = page >> 8;
1535 if (chip->options & NAND_ROW_ADDR_3)
1536 addrs[naddrs++] = page >> 16;
1538 instrs[2].ctx.addr.naddrs = naddrs;
1540 /* Drop the last two instructions if we're not programming the page. */
1543 /* Also drop the DATA_OUT instruction if empty. */
1548 if (mtd->writesize <= 512) {
1550 * Small pages need some more tweaking: we have to adjust the
1551 * first instruction depending on the page offset we're trying
1554 if (offset_in_page >= mtd->writesize)
1555 instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
1556 else if (offset_in_page >= 256 &&
1557 !(chip->options & NAND_BUSWIDTH_16))
1558 instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
1561 * Drop the first command if we're dealing with a large page
1568 return nand_exec_op(chip, &op);
1572 * nand_prog_page_begin_op - starts a PROG PAGE operation
1573 * @chip: The NAND chip
1574 * @page: page to write
1575 * @offset_in_page: offset within the page
1576 * @buf: buffer containing the data to write to the page
1577 * @len: length of the buffer
1579 * This function issues the first half of a PROG PAGE operation.
1580 * This function does not select/unselect the CS line.
1582 * Returns 0 on success, a negative error code otherwise.
1584 int nand_prog_page_begin_op(struct nand_chip *chip, unsigned int page,
1585 unsigned int offset_in_page, const void *buf,
1588 struct mtd_info *mtd = nand_to_mtd(chip);
1593 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1596 if (nand_has_exec_op(chip))
1597 return nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1600 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page);
1603 chip->legacy.write_buf(chip, buf, len);
1607 EXPORT_SYMBOL_GPL(nand_prog_page_begin_op);
1610 * nand_prog_page_end_op - ends a PROG PAGE operation
1611 * @chip: The NAND chip
1613 * This function issues the second half of a PROG PAGE operation.
1614 * This function does not select/unselect the CS line.
1616 * Returns 0 on success, a negative error code otherwise.
1618 int nand_prog_page_end_op(struct nand_chip *chip)
1623 if (nand_has_exec_op(chip)) {
1624 const struct nand_interface_config *conf =
1625 nand_get_interface_config(chip);
1626 struct nand_op_instr instrs[] = {
1627 NAND_OP_CMD(NAND_CMD_PAGEPROG,
1628 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1629 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max),
1632 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1634 ret = nand_exec_op(chip, &op);
1638 ret = nand_status_op(chip, &status);
1642 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1643 ret = chip->legacy.waitfunc(chip);
1650 if (status & NAND_STATUS_FAIL)
1655 EXPORT_SYMBOL_GPL(nand_prog_page_end_op);
1658 * nand_prog_page_op - Do a full PROG PAGE operation
1659 * @chip: The NAND chip
1660 * @page: page to write
1661 * @offset_in_page: offset within the page
1662 * @buf: buffer containing the data to write to the page
1663 * @len: length of the buffer
1665 * This function issues a full PROG PAGE operation.
1666 * This function does not select/unselect the CS line.
1668 * Returns 0 on success, a negative error code otherwise.
1670 int nand_prog_page_op(struct nand_chip *chip, unsigned int page,
1671 unsigned int offset_in_page, const void *buf,
1674 struct mtd_info *mtd = nand_to_mtd(chip);
1681 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1684 if (nand_has_exec_op(chip)) {
1685 ret = nand_exec_prog_page_op(chip, page, offset_in_page, buf,
1690 ret = nand_status_op(chip, &status);
1694 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page,
1696 chip->legacy.write_buf(chip, buf, len);
1697 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1698 ret = chip->legacy.waitfunc(chip);
1705 if (status & NAND_STATUS_FAIL)
1710 EXPORT_SYMBOL_GPL(nand_prog_page_op);
1713 * nand_change_write_column_op - Do a CHANGE WRITE COLUMN operation
1714 * @chip: The NAND chip
1715 * @offset_in_page: offset within the page
1716 * @buf: buffer containing the data to send to the NAND
1717 * @len: length of the buffer
1718 * @force_8bit: force 8-bit bus access
1720 * This function issues a CHANGE WRITE COLUMN operation.
1721 * This function does not select/unselect the CS line.
1723 * Returns 0 on success, a negative error code otherwise.
1725 int nand_change_write_column_op(struct nand_chip *chip,
1726 unsigned int offset_in_page,
1727 const void *buf, unsigned int len,
1730 struct mtd_info *mtd = nand_to_mtd(chip);
1735 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1738 /* Small page NANDs do not support column change. */
1739 if (mtd->writesize <= 512)
1742 if (nand_has_exec_op(chip)) {
1743 const struct nand_interface_config *conf =
1744 nand_get_interface_config(chip);
1746 struct nand_op_instr instrs[] = {
1747 NAND_OP_CMD(NAND_CMD_RNDIN, 0),
1748 NAND_OP_ADDR(2, addrs, NAND_COMMON_TIMING_NS(conf, tCCS_min)),
1749 NAND_OP_DATA_OUT(len, buf, 0),
1751 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1754 ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
1758 instrs[2].ctx.data.force_8bit = force_8bit;
1760 /* Drop the DATA_OUT instruction if len is set to 0. */
1764 return nand_exec_op(chip, &op);
1767 chip->legacy.cmdfunc(chip, NAND_CMD_RNDIN, offset_in_page, -1);
1769 chip->legacy.write_buf(chip, buf, len);
1773 EXPORT_SYMBOL_GPL(nand_change_write_column_op);
1776 * nand_readid_op - Do a READID operation
1777 * @chip: The NAND chip
1778 * @addr: address cycle to pass after the READID command
1779 * @buf: buffer used to store the ID
1780 * @len: length of the buffer
1782 * This function sends a READID command and reads back the ID returned by the
1784 * This function does not select/unselect the CS line.
1786 * Returns 0 on success, a negative error code otherwise.
1788 int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf,
1792 u8 *id = buf, *ddrbuf = NULL;
1797 if (nand_has_exec_op(chip)) {
1798 const struct nand_interface_config *conf =
1799 nand_get_interface_config(chip);
1800 struct nand_op_instr instrs[] = {
1801 NAND_OP_CMD(NAND_CMD_READID, 0),
1802 NAND_OP_ADDR(1, &addr,
1803 NAND_COMMON_TIMING_NS(conf, tADL_min)),
1804 NAND_OP_8BIT_DATA_IN(len, buf, 0),
1806 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1809 /* READ_ID data bytes are received twice in NV-DDR mode */
1810 if (len && nand_interface_is_nvddr(conf)) {
1811 ddrbuf = kzalloc(len * 2, GFP_KERNEL);
1815 instrs[2].ctx.data.len *= 2;
1816 instrs[2].ctx.data.buf.in = ddrbuf;
1819 /* Drop the DATA_IN instruction if len is set to 0. */
1823 ret = nand_exec_op(chip, &op);
1824 if (!ret && len && nand_interface_is_nvddr(conf)) {
1825 for (i = 0; i < len; i++)
1826 id[i] = ddrbuf[i * 2];
1834 chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1);
1836 for (i = 0; i < len; i++)
1837 id[i] = chip->legacy.read_byte(chip);
1841 EXPORT_SYMBOL_GPL(nand_readid_op);
1844 * nand_status_op - Do a STATUS operation
1845 * @chip: The NAND chip
1846 * @status: out variable to store the NAND status
1848 * This function sends a STATUS command and reads back the status returned by
1850 * This function does not select/unselect the CS line.
1852 * Returns 0 on success, a negative error code otherwise.
1854 int nand_status_op(struct nand_chip *chip, u8 *status)
1856 if (nand_has_exec_op(chip)) {
1857 const struct nand_interface_config *conf =
1858 nand_get_interface_config(chip);
1860 struct nand_op_instr instrs[] = {
1861 NAND_OP_CMD(NAND_CMD_STATUS,
1862 NAND_COMMON_TIMING_NS(conf, tADL_min)),
1863 NAND_OP_8BIT_DATA_IN(1, status, 0),
1865 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1868 /* The status data byte will be received twice in NV-DDR mode */
1869 if (status && nand_interface_is_nvddr(conf)) {
1870 instrs[1].ctx.data.len *= 2;
1871 instrs[1].ctx.data.buf.in = ddrstatus;
1877 ret = nand_exec_op(chip, &op);
1878 if (!ret && status && nand_interface_is_nvddr(conf))
1879 *status = ddrstatus[0];
1884 chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1);
1886 *status = chip->legacy.read_byte(chip);
1890 EXPORT_SYMBOL_GPL(nand_status_op);
1893 * nand_exit_status_op - Exit a STATUS operation
1894 * @chip: The NAND chip
1896 * This function sends a READ0 command to cancel the effect of the STATUS
1897 * command to avoid reading only the status until a new read command is sent.
1899 * This function does not select/unselect the CS line.
1901 * Returns 0 on success, a negative error code otherwise.
1903 int nand_exit_status_op(struct nand_chip *chip)
1905 if (nand_has_exec_op(chip)) {
1906 struct nand_op_instr instrs[] = {
1907 NAND_OP_CMD(NAND_CMD_READ0, 0),
1909 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1911 return nand_exec_op(chip, &op);
1914 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, -1, -1);
1918 EXPORT_SYMBOL_GPL(nand_exit_status_op);
1921 * nand_erase_op - Do an erase operation
1922 * @chip: The NAND chip
1923 * @eraseblock: block to erase
1925 * This function sends an ERASE command and waits for the NAND to be ready
1927 * This function does not select/unselect the CS line.
1929 * Returns 0 on success, a negative error code otherwise.
1931 int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock)
1933 unsigned int page = eraseblock <<
1934 (chip->phys_erase_shift - chip->page_shift);
1938 if (nand_has_exec_op(chip)) {
1939 const struct nand_interface_config *conf =
1940 nand_get_interface_config(chip);
1941 u8 addrs[3] = { page, page >> 8, page >> 16 };
1942 struct nand_op_instr instrs[] = {
1943 NAND_OP_CMD(NAND_CMD_ERASE1, 0),
1944 NAND_OP_ADDR(2, addrs, 0),
1945 NAND_OP_CMD(NAND_CMD_ERASE2,
1946 NAND_COMMON_TIMING_NS(conf, tWB_max)),
1947 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tBERS_max),
1950 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1952 if (chip->options & NAND_ROW_ADDR_3)
1953 instrs[1].ctx.addr.naddrs++;
1955 ret = nand_exec_op(chip, &op);
1959 ret = nand_status_op(chip, &status);
1963 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE1, -1, page);
1964 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE2, -1, -1);
1966 ret = chip->legacy.waitfunc(chip);
1973 if (status & NAND_STATUS_FAIL)
1978 EXPORT_SYMBOL_GPL(nand_erase_op);
1981 * nand_set_features_op - Do a SET FEATURES operation
1982 * @chip: The NAND chip
1983 * @feature: feature id
1984 * @data: 4 bytes of data
1986 * This function sends a SET FEATURES command and waits for the NAND to be
1987 * ready before returning.
1988 * This function does not select/unselect the CS line.
1990 * Returns 0 on success, a negative error code otherwise.
1992 static int nand_set_features_op(struct nand_chip *chip, u8 feature,
1995 const u8 *params = data;
1998 if (nand_has_exec_op(chip)) {
1999 const struct nand_interface_config *conf =
2000 nand_get_interface_config(chip);
2001 struct nand_op_instr instrs[] = {
2002 NAND_OP_CMD(NAND_CMD_SET_FEATURES, 0),
2003 NAND_OP_ADDR(1, &feature, NAND_COMMON_TIMING_NS(conf,
2005 NAND_OP_8BIT_DATA_OUT(ONFI_SUBFEATURE_PARAM_LEN, data,
2006 NAND_COMMON_TIMING_NS(conf,
2008 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max),
2011 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2013 return nand_exec_op(chip, &op);
2016 chip->legacy.cmdfunc(chip, NAND_CMD_SET_FEATURES, feature, -1);
2017 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
2018 chip->legacy.write_byte(chip, params[i]);
2020 ret = chip->legacy.waitfunc(chip);
2024 if (ret & NAND_STATUS_FAIL)
2031 * nand_get_features_op - Do a GET FEATURES operation
2032 * @chip: The NAND chip
2033 * @feature: feature id
2034 * @data: 4 bytes of data
2036 * This function sends a GET FEATURES command and waits for the NAND to be
2037 * ready before returning.
2038 * This function does not select/unselect the CS line.
2040 * Returns 0 on success, a negative error code otherwise.
2042 static int nand_get_features_op(struct nand_chip *chip, u8 feature,
2045 u8 *params = data, ddrbuf[ONFI_SUBFEATURE_PARAM_LEN * 2];
2048 if (nand_has_exec_op(chip)) {
2049 const struct nand_interface_config *conf =
2050 nand_get_interface_config(chip);
2051 struct nand_op_instr instrs[] = {
2052 NAND_OP_CMD(NAND_CMD_GET_FEATURES, 0),
2053 NAND_OP_ADDR(1, &feature,
2054 NAND_COMMON_TIMING_NS(conf, tWB_max)),
2055 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max),
2056 NAND_COMMON_TIMING_NS(conf, tRR_min)),
2057 NAND_OP_8BIT_DATA_IN(ONFI_SUBFEATURE_PARAM_LEN,
2060 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2063 /* GET_FEATURE data bytes are received twice in NV-DDR mode */
2064 if (nand_interface_is_nvddr(conf)) {
2065 instrs[3].ctx.data.len *= 2;
2066 instrs[3].ctx.data.buf.in = ddrbuf;
2069 ret = nand_exec_op(chip, &op);
2070 if (nand_interface_is_nvddr(conf)) {
2071 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; i++)
2072 params[i] = ddrbuf[i * 2];
2078 chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1);
2079 for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
2080 params[i] = chip->legacy.read_byte(chip);
2085 static int nand_wait_rdy_op(struct nand_chip *chip, unsigned int timeout_ms,
2086 unsigned int delay_ns)
2088 if (nand_has_exec_op(chip)) {
2089 struct nand_op_instr instrs[] = {
2090 NAND_OP_WAIT_RDY(PSEC_TO_MSEC(timeout_ms),
2091 PSEC_TO_NSEC(delay_ns)),
2093 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2095 return nand_exec_op(chip, &op);
2098 /* Apply delay or wait for ready/busy pin */
2099 if (!chip->legacy.dev_ready)
2100 udelay(chip->legacy.chip_delay);
2102 nand_wait_ready(chip);
2108 * nand_reset_op - Do a reset operation
2109 * @chip: The NAND chip
2111 * This function sends a RESET command and waits for the NAND to be ready
2113 * This function does not select/unselect the CS line.
2115 * Returns 0 on success, a negative error code otherwise.
2117 int nand_reset_op(struct nand_chip *chip)
2119 if (nand_has_exec_op(chip)) {
2120 const struct nand_interface_config *conf =
2121 nand_get_interface_config(chip);
2122 struct nand_op_instr instrs[] = {
2123 NAND_OP_CMD(NAND_CMD_RESET,
2124 NAND_COMMON_TIMING_NS(conf, tWB_max)),
2125 NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tRST_max),
2128 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2130 return nand_exec_op(chip, &op);
2133 chip->legacy.cmdfunc(chip, NAND_CMD_RESET, -1, -1);
2137 EXPORT_SYMBOL_GPL(nand_reset_op);
2140 * nand_read_data_op - Read data from the NAND
2141 * @chip: The NAND chip
2142 * @buf: buffer used to store the data
2143 * @len: length of the buffer
2144 * @force_8bit: force 8-bit bus access
2145 * @check_only: do not actually run the command, only checks if the
2146 * controller driver supports it
2148 * This function does a raw data read on the bus. Usually used after launching
2149 * another NAND operation like nand_read_page_op().
2150 * This function does not select/unselect the CS line.
2152 * Returns 0 on success, a negative error code otherwise.
2154 int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
2155 bool force_8bit, bool check_only)
2160 if (nand_has_exec_op(chip)) {
2161 const struct nand_interface_config *conf =
2162 nand_get_interface_config(chip);
2163 struct nand_op_instr instrs[] = {
2164 NAND_OP_DATA_IN(len, buf, 0),
2166 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2170 instrs[0].ctx.data.force_8bit = force_8bit;
2173 * Parameter payloads (ID, status, features, etc) do not go
2174 * through the same pipeline as regular data, hence the
2175 * force_8bit flag must be set and this also indicates that in
2176 * case NV-DDR timings are being used the data will be received
2179 if (force_8bit && nand_interface_is_nvddr(conf)) {
2180 ddrbuf = kzalloc(len * 2, GFP_KERNEL);
2184 instrs[0].ctx.data.len *= 2;
2185 instrs[0].ctx.data.buf.in = ddrbuf;
2189 ret = nand_check_op(chip, &op);
2194 ret = nand_exec_op(chip, &op);
2195 if (!ret && force_8bit && nand_interface_is_nvddr(conf)) {
2198 for (i = 0; i < len; i++)
2199 dst[i] = ddrbuf[i * 2];
2214 for (i = 0; i < len; i++)
2215 p[i] = chip->legacy.read_byte(chip);
2217 chip->legacy.read_buf(chip, buf, len);
2222 EXPORT_SYMBOL_GPL(nand_read_data_op);
2225 * nand_write_data_op - Write data from the NAND
2226 * @chip: The NAND chip
2227 * @buf: buffer containing the data to send on the bus
2228 * @len: length of the buffer
2229 * @force_8bit: force 8-bit bus access
2231 * This function does a raw data write on the bus. Usually used after launching
2232 * another NAND operation like nand_write_page_begin_op().
2233 * This function does not select/unselect the CS line.
2235 * Returns 0 on success, a negative error code otherwise.
2237 int nand_write_data_op(struct nand_chip *chip, const void *buf,
2238 unsigned int len, bool force_8bit)
2243 if (nand_has_exec_op(chip)) {
2244 struct nand_op_instr instrs[] = {
2245 NAND_OP_DATA_OUT(len, buf, 0),
2247 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2249 instrs[0].ctx.data.force_8bit = force_8bit;
2251 return nand_exec_op(chip, &op);
2258 for (i = 0; i < len; i++)
2259 chip->legacy.write_byte(chip, p[i]);
2261 chip->legacy.write_buf(chip, buf, len);
2266 EXPORT_SYMBOL_GPL(nand_write_data_op);
2269 * struct nand_op_parser_ctx - Context used by the parser
2270 * @instrs: array of all the instructions that must be addressed
2271 * @ninstrs: length of the @instrs array
2272 * @subop: Sub-operation to be passed to the NAND controller
2274 * This structure is used by the core to split NAND operations into
2275 * sub-operations that can be handled by the NAND controller.
2277 struct nand_op_parser_ctx {
2278 const struct nand_op_instr *instrs;
2279 unsigned int ninstrs;
2280 struct nand_subop subop;
2284 * nand_op_parser_must_split_instr - Checks if an instruction must be split
2285 * @pat: the parser pattern element that matches @instr
2286 * @instr: pointer to the instruction to check
2287 * @start_offset: this is an in/out parameter. If @instr has already been
2288 * split, then @start_offset is the offset from which to start
2289 * (either an address cycle or an offset in the data buffer).
2290 * Conversely, if the function returns true (ie. instr must be
2291 * split), this parameter is updated to point to the first
2292 * data/address cycle that has not been taken care of.
2294 * Some NAND controllers are limited and cannot send X address cycles with a
2295 * unique operation, or cannot read/write more than Y bytes at the same time.
2296 * In this case, split the instruction that does not fit in a single
2297 * controller-operation into two or more chunks.
2299 * Returns true if the instruction must be split, false otherwise.
2300 * The @start_offset parameter is also updated to the offset at which the next
2301 * bundle of instruction must start (if an address or a data instruction).
2304 nand_op_parser_must_split_instr(const struct nand_op_parser_pattern_elem *pat,
2305 const struct nand_op_instr *instr,
2306 unsigned int *start_offset)
2308 switch (pat->type) {
2309 case NAND_OP_ADDR_INSTR:
2310 if (!pat->ctx.addr.maxcycles)
2313 if (instr->ctx.addr.naddrs - *start_offset >
2314 pat->ctx.addr.maxcycles) {
2315 *start_offset += pat->ctx.addr.maxcycles;
2320 case NAND_OP_DATA_IN_INSTR:
2321 case NAND_OP_DATA_OUT_INSTR:
2322 if (!pat->ctx.data.maxlen)
2325 if (instr->ctx.data.len - *start_offset >
2326 pat->ctx.data.maxlen) {
2327 *start_offset += pat->ctx.data.maxlen;
2340 * nand_op_parser_match_pat - Checks if a pattern matches the instructions
2341 * remaining in the parser context
2342 * @pat: the pattern to test
2343 * @ctx: the parser context structure to match with the pattern @pat
2345 * Check if @pat matches the set or a sub-set of instructions remaining in @ctx.
2346 * Returns true if this is the case, false ortherwise. When true is returned,
2347 * @ctx->subop is updated with the set of instructions to be passed to the
2348 * controller driver.
2351 nand_op_parser_match_pat(const struct nand_op_parser_pattern *pat,
2352 struct nand_op_parser_ctx *ctx)
2354 unsigned int instr_offset = ctx->subop.first_instr_start_off;
2355 const struct nand_op_instr *end = ctx->instrs + ctx->ninstrs;
2356 const struct nand_op_instr *instr = ctx->subop.instrs;
2357 unsigned int i, ninstrs;
2359 for (i = 0, ninstrs = 0; i < pat->nelems && instr < end; i++) {
2361 * The pattern instruction does not match the operation
2362 * instruction. If the instruction is marked optional in the
2363 * pattern definition, we skip the pattern element and continue
2364 * to the next one. If the element is mandatory, there's no
2365 * match and we can return false directly.
2367 if (instr->type != pat->elems[i].type) {
2368 if (!pat->elems[i].optional)
2375 * Now check the pattern element constraints. If the pattern is
2376 * not able to handle the whole instruction in a single step,
2377 * we have to split it.
2378 * The last_instr_end_off value comes back updated to point to
2379 * the position where we have to split the instruction (the
2380 * start of the next subop chunk).
2382 if (nand_op_parser_must_split_instr(&pat->elems[i], instr,
2395 * This can happen if all instructions of a pattern are optional.
2396 * Still, if there's not at least one instruction handled by this
2397 * pattern, this is not a match, and we should try the next one (if
2404 * We had a match on the pattern head, but the pattern may be longer
2405 * than the instructions we're asked to execute. We need to make sure
2406 * there's no mandatory elements in the pattern tail.
2408 for (; i < pat->nelems; i++) {
2409 if (!pat->elems[i].optional)
2414 * We have a match: update the subop structure accordingly and return
2417 ctx->subop.ninstrs = ninstrs;
2418 ctx->subop.last_instr_end_off = instr_offset;
2423 #if IS_ENABLED(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG)
2424 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2426 const struct nand_op_instr *instr;
2430 pr_debug("executing subop (CS%d):\n", ctx->subop.cs);
2432 for (i = 0; i < ctx->ninstrs; i++) {
2433 instr = &ctx->instrs[i];
2435 if (instr == &ctx->subop.instrs[0])
2438 nand_op_trace(prefix, instr);
2440 if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1])
2445 static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
2451 static int nand_op_parser_cmp_ctx(const struct nand_op_parser_ctx *a,
2452 const struct nand_op_parser_ctx *b)
2454 if (a->subop.ninstrs < b->subop.ninstrs)
2456 else if (a->subop.ninstrs > b->subop.ninstrs)
2459 if (a->subop.last_instr_end_off < b->subop.last_instr_end_off)
2461 else if (a->subop.last_instr_end_off > b->subop.last_instr_end_off)
2468 * nand_op_parser_exec_op - exec_op parser
2469 * @chip: the NAND chip
2470 * @parser: patterns description provided by the controller driver
2471 * @op: the NAND operation to address
2472 * @check_only: when true, the function only checks if @op can be handled but
2473 * does not execute the operation
2475 * Helper function designed to ease integration of NAND controller drivers that
2476 * only support a limited set of instruction sequences. The supported sequences
2477 * are described in @parser, and the framework takes care of splitting @op into
2478 * multiple sub-operations (if required) and pass them back to the ->exec()
2479 * callback of the matching pattern if @check_only is set to false.
2481 * NAND controller drivers should call this function from their own ->exec_op()
2484 * Returns 0 on success, a negative error code otherwise. A failure can be
2485 * caused by an unsupported operation (none of the supported patterns is able
2486 * to handle the requested operation), or an error returned by one of the
2487 * matching pattern->exec() hook.
2489 int nand_op_parser_exec_op(struct nand_chip *chip,
2490 const struct nand_op_parser *parser,
2491 const struct nand_operation *op, bool check_only)
2493 struct nand_op_parser_ctx ctx = {
2495 .subop.instrs = op->instrs,
2496 .instrs = op->instrs,
2497 .ninstrs = op->ninstrs,
2501 while (ctx.subop.instrs < op->instrs + op->ninstrs) {
2502 const struct nand_op_parser_pattern *pattern;
2503 struct nand_op_parser_ctx best_ctx;
2504 int ret, best_pattern = -1;
2506 for (i = 0; i < parser->npatterns; i++) {
2507 struct nand_op_parser_ctx test_ctx = ctx;
2509 pattern = &parser->patterns[i];
2510 if (!nand_op_parser_match_pat(pattern, &test_ctx))
2513 if (best_pattern >= 0 &&
2514 nand_op_parser_cmp_ctx(&test_ctx, &best_ctx) <= 0)
2518 best_ctx = test_ctx;
2521 if (best_pattern < 0) {
2522 pr_debug("->exec_op() parser: pattern not found!\n");
2527 nand_op_parser_trace(&ctx);
2530 pattern = &parser->patterns[best_pattern];
2531 ret = pattern->exec(chip, &ctx.subop);
2537 * Update the context structure by pointing to the start of the
2540 ctx.subop.instrs = ctx.subop.instrs + ctx.subop.ninstrs;
2541 if (ctx.subop.last_instr_end_off)
2542 ctx.subop.instrs -= 1;
2544 ctx.subop.first_instr_start_off = ctx.subop.last_instr_end_off;
2549 EXPORT_SYMBOL_GPL(nand_op_parser_exec_op);
2551 static bool nand_instr_is_data(const struct nand_op_instr *instr)
2553 return instr && (instr->type == NAND_OP_DATA_IN_INSTR ||
2554 instr->type == NAND_OP_DATA_OUT_INSTR);
2557 static bool nand_subop_instr_is_valid(const struct nand_subop *subop,
2558 unsigned int instr_idx)
2560 return subop && instr_idx < subop->ninstrs;
2563 static unsigned int nand_subop_get_start_off(const struct nand_subop *subop,
2564 unsigned int instr_idx)
2569 return subop->first_instr_start_off;
2573 * nand_subop_get_addr_start_off - Get the start offset in an address array
2574 * @subop: The entire sub-operation
2575 * @instr_idx: Index of the instruction inside the sub-operation
2577 * During driver development, one could be tempted to directly use the
2578 * ->addr.addrs field of address instructions. This is wrong as address
2579 * instructions might be split.
2581 * Given an address instruction, returns the offset of the first cycle to issue.
2583 unsigned int nand_subop_get_addr_start_off(const struct nand_subop *subop,
2584 unsigned int instr_idx)
2586 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2587 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2590 return nand_subop_get_start_off(subop, instr_idx);
2592 EXPORT_SYMBOL_GPL(nand_subop_get_addr_start_off);
2595 * nand_subop_get_num_addr_cyc - Get the remaining address cycles to assert
2596 * @subop: The entire sub-operation
2597 * @instr_idx: Index of the instruction inside the sub-operation
2599 * During driver development, one could be tempted to directly use the
2600 * ->addr->naddrs field of a data instruction. This is wrong as instructions
2603 * Given an address instruction, returns the number of address cycle to issue.
2605 unsigned int nand_subop_get_num_addr_cyc(const struct nand_subop *subop,
2606 unsigned int instr_idx)
2608 int start_off, end_off;
2610 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2611 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2614 start_off = nand_subop_get_addr_start_off(subop, instr_idx);
2616 if (instr_idx == subop->ninstrs - 1 &&
2617 subop->last_instr_end_off)
2618 end_off = subop->last_instr_end_off;
2620 end_off = subop->instrs[instr_idx].ctx.addr.naddrs;
2622 return end_off - start_off;
2624 EXPORT_SYMBOL_GPL(nand_subop_get_num_addr_cyc);
2627 * nand_subop_get_data_start_off - Get the start offset in a data array
2628 * @subop: The entire sub-operation
2629 * @instr_idx: Index of the instruction inside the sub-operation
2631 * During driver development, one could be tempted to directly use the
2632 * ->data->buf.{in,out} field of data instructions. This is wrong as data
2633 * instructions might be split.
2635 * Given a data instruction, returns the offset to start from.
2637 unsigned int nand_subop_get_data_start_off(const struct nand_subop *subop,
2638 unsigned int instr_idx)
2640 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2641 !nand_instr_is_data(&subop->instrs[instr_idx])))
2644 return nand_subop_get_start_off(subop, instr_idx);
2646 EXPORT_SYMBOL_GPL(nand_subop_get_data_start_off);
2649 * nand_subop_get_data_len - Get the number of bytes to retrieve
2650 * @subop: The entire sub-operation
2651 * @instr_idx: Index of the instruction inside the sub-operation
2653 * During driver development, one could be tempted to directly use the
2654 * ->data->len field of a data instruction. This is wrong as data instructions
2657 * Returns the length of the chunk of data to send/receive.
2659 unsigned int nand_subop_get_data_len(const struct nand_subop *subop,
2660 unsigned int instr_idx)
2662 int start_off = 0, end_off;
2664 if (WARN_ON(!nand_subop_instr_is_valid(subop, instr_idx) ||
2665 !nand_instr_is_data(&subop->instrs[instr_idx])))
2668 start_off = nand_subop_get_data_start_off(subop, instr_idx);
2670 if (instr_idx == subop->ninstrs - 1 &&
2671 subop->last_instr_end_off)
2672 end_off = subop->last_instr_end_off;
2674 end_off = subop->instrs[instr_idx].ctx.data.len;
2676 return end_off - start_off;
2678 EXPORT_SYMBOL_GPL(nand_subop_get_data_len);
2681 * nand_reset - Reset and initialize a NAND device
2682 * @chip: The NAND chip
2683 * @chipnr: Internal die id
2685 * Save the timings data structure, then apply SDR timings mode 0 (see
2686 * nand_reset_interface for details), do the reset operation, and apply
2687 * back the previous timings.
2689 * Returns 0 on success, a negative error code otherwise.
2691 int nand_reset(struct nand_chip *chip, int chipnr)
2695 ret = nand_reset_interface(chip, chipnr);
2700 * The CS line has to be released before we can apply the new NAND
2701 * interface settings, hence this weird nand_select_target()
2702 * nand_deselect_target() dance.
2704 nand_select_target(chip, chipnr);
2705 ret = nand_reset_op(chip);
2706 nand_deselect_target(chip);
2710 ret = nand_setup_interface(chip, chipnr);
2716 EXPORT_SYMBOL_GPL(nand_reset);
2719 * nand_get_features - wrapper to perform a GET_FEATURE
2720 * @chip: NAND chip info structure
2721 * @addr: feature address
2722 * @subfeature_param: the subfeature parameters, a four bytes array
2724 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2725 * operation cannot be handled.
2727 int nand_get_features(struct nand_chip *chip, int addr,
2728 u8 *subfeature_param)
2730 if (!nand_supports_get_features(chip, addr))
2733 if (chip->legacy.get_features)
2734 return chip->legacy.get_features(chip, addr, subfeature_param);
2736 return nand_get_features_op(chip, addr, subfeature_param);
2740 * nand_set_features - wrapper to perform a SET_FEATURE
2741 * @chip: NAND chip info structure
2742 * @addr: feature address
2743 * @subfeature_param: the subfeature parameters, a four bytes array
2745 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2746 * operation cannot be handled.
2748 int nand_set_features(struct nand_chip *chip, int addr,
2749 u8 *subfeature_param)
2751 if (!nand_supports_set_features(chip, addr))
2754 if (chip->legacy.set_features)
2755 return chip->legacy.set_features(chip, addr, subfeature_param);
2757 return nand_set_features_op(chip, addr, subfeature_param);
2761 * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
2762 * @buf: buffer to test
2763 * @len: buffer length
2764 * @bitflips_threshold: maximum number of bitflips
2766 * Check if a buffer contains only 0xff, which means the underlying region
2767 * has been erased and is ready to be programmed.
2768 * The bitflips_threshold specify the maximum number of bitflips before
2769 * considering the region is not erased.
2770 * Note: The logic of this function has been extracted from the memweight
2771 * implementation, except that nand_check_erased_buf function exit before
2772 * testing the whole buffer if the number of bitflips exceed the
2773 * bitflips_threshold value.
2775 * Returns a positive number of bitflips less than or equal to
2776 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2779 static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold)
2781 const unsigned char *bitmap = buf;
2785 for (; len && ((uintptr_t)bitmap) % sizeof(long);
2787 weight = hweight8(*bitmap);
2788 bitflips += BITS_PER_BYTE - weight;
2789 if (unlikely(bitflips > bitflips_threshold))
2793 for (; len >= sizeof(long);
2794 len -= sizeof(long), bitmap += sizeof(long)) {
2795 unsigned long d = *((unsigned long *)bitmap);
2798 weight = hweight_long(d);
2799 bitflips += BITS_PER_LONG - weight;
2800 if (unlikely(bitflips > bitflips_threshold))
2804 for (; len > 0; len--, bitmap++) {
2805 weight = hweight8(*bitmap);
2806 bitflips += BITS_PER_BYTE - weight;
2807 if (unlikely(bitflips > bitflips_threshold))
2815 * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
2817 * @data: data buffer to test
2818 * @datalen: data length
2820 * @ecclen: ECC length
2821 * @extraoob: extra OOB buffer
2822 * @extraooblen: extra OOB length
2823 * @bitflips_threshold: maximum number of bitflips
2825 * Check if a data buffer and its associated ECC and OOB data contains only
2826 * 0xff pattern, which means the underlying region has been erased and is
2827 * ready to be programmed.
2828 * The bitflips_threshold specify the maximum number of bitflips before
2829 * considering the region as not erased.
2832 * 1/ ECC algorithms are working on pre-defined block sizes which are usually
2833 * different from the NAND page size. When fixing bitflips, ECC engines will
2834 * report the number of errors per chunk, and the NAND core infrastructure
2835 * expect you to return the maximum number of bitflips for the whole page.
2836 * This is why you should always use this function on a single chunk and
2837 * not on the whole page. After checking each chunk you should update your
2838 * max_bitflips value accordingly.
2839 * 2/ When checking for bitflips in erased pages you should not only check
2840 * the payload data but also their associated ECC data, because a user might
2841 * have programmed almost all bits to 1 but a few. In this case, we
2842 * shouldn't consider the chunk as erased, and checking ECC bytes prevent
2844 * 3/ The extraoob argument is optional, and should be used if some of your OOB
2845 * data are protected by the ECC engine.
2846 * It could also be used if you support subpages and want to attach some
2847 * extra OOB data to an ECC chunk.
2849 * Returns a positive number of bitflips less than or equal to
2850 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2851 * threshold. In case of success, the passed buffers are filled with 0xff.
2853 int nand_check_erased_ecc_chunk(void *data, int datalen,
2854 void *ecc, int ecclen,
2855 void *extraoob, int extraooblen,
2856 int bitflips_threshold)
2858 int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0;
2860 data_bitflips = nand_check_erased_buf(data, datalen,
2861 bitflips_threshold);
2862 if (data_bitflips < 0)
2863 return data_bitflips;
2865 bitflips_threshold -= data_bitflips;
2867 ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
2868 if (ecc_bitflips < 0)
2869 return ecc_bitflips;
2871 bitflips_threshold -= ecc_bitflips;
2873 extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen,
2874 bitflips_threshold);
2875 if (extraoob_bitflips < 0)
2876 return extraoob_bitflips;
2879 memset(data, 0xff, datalen);
2882 memset(ecc, 0xff, ecclen);
2884 if (extraoob_bitflips)
2885 memset(extraoob, 0xff, extraooblen);
2887 return data_bitflips + ecc_bitflips + extraoob_bitflips;
2889 EXPORT_SYMBOL(nand_check_erased_ecc_chunk);
2892 * nand_read_page_raw_notsupp - dummy read raw page function
2893 * @chip: nand chip info structure
2894 * @buf: buffer to store read data
2895 * @oob_required: caller requires OOB data read to chip->oob_poi
2896 * @page: page number to read
2898 * Returns -ENOTSUPP unconditionally.
2900 int nand_read_page_raw_notsupp(struct nand_chip *chip, u8 *buf,
2901 int oob_required, int page)
2907 * nand_read_page_raw - [INTERN] read raw page data without ecc
2908 * @chip: nand chip info structure
2909 * @buf: buffer to store read data
2910 * @oob_required: caller requires OOB data read to chip->oob_poi
2911 * @page: page number to read
2913 * Not for syndrome calculating ECC controllers, which use a special oob layout.
2915 int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required,
2918 struct mtd_info *mtd = nand_to_mtd(chip);
2921 ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize);
2926 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
2934 EXPORT_SYMBOL(nand_read_page_raw);
2937 * nand_monolithic_read_page_raw - Monolithic page read in raw mode
2938 * @chip: NAND chip info structure
2939 * @buf: buffer to store read data
2940 * @oob_required: caller requires OOB data read to chip->oob_poi
2941 * @page: page number to read
2943 * This is a raw page read, ie. without any error detection/correction.
2944 * Monolithic means we are requesting all the relevant data (main plus
2945 * eventually OOB) to be loaded in the NAND cache and sent over the
2946 * bus (from the NAND chip to the NAND controller) in a single
2947 * operation. This is an alternative to nand_read_page_raw(), which
2948 * first reads the main data, and if the OOB data is requested too,
2949 * then reads more data on the bus.
2951 int nand_monolithic_read_page_raw(struct nand_chip *chip, u8 *buf,
2952 int oob_required, int page)
2954 struct mtd_info *mtd = nand_to_mtd(chip);
2955 unsigned int size = mtd->writesize;
2960 size += mtd->oobsize;
2962 if (buf != chip->data_buf)
2963 read_buf = nand_get_data_buf(chip);
2966 ret = nand_read_page_op(chip, page, 0, read_buf, size);
2970 if (buf != chip->data_buf)
2971 memcpy(buf, read_buf, mtd->writesize);
2975 EXPORT_SYMBOL(nand_monolithic_read_page_raw);
2978 * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
2979 * @chip: nand chip info structure
2980 * @buf: buffer to store read data
2981 * @oob_required: caller requires OOB data read to chip->oob_poi
2982 * @page: page number to read
2984 * We need a special oob layout and handling even when OOB isn't used.
2986 static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
2987 int oob_required, int page)
2989 struct mtd_info *mtd = nand_to_mtd(chip);
2990 int eccsize = chip->ecc.size;
2991 int eccbytes = chip->ecc.bytes;
2992 uint8_t *oob = chip->oob_poi;
2993 int steps, size, ret;
2995 ret = nand_read_page_op(chip, page, 0, NULL, 0);
2999 for (steps = chip->ecc.steps; steps > 0; steps--) {
3000 ret = nand_read_data_op(chip, buf, eccsize, false, false);
3006 if (chip->ecc.prepad) {
3007 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3012 oob += chip->ecc.prepad;
3015 ret = nand_read_data_op(chip, oob, eccbytes, false, false);
3021 if (chip->ecc.postpad) {
3022 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3027 oob += chip->ecc.postpad;
3031 size = mtd->oobsize - (oob - chip->oob_poi);
3033 ret = nand_read_data_op(chip, oob, size, false, false);
3042 * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function
3043 * @chip: nand chip info structure
3044 * @buf: buffer to store read data
3045 * @oob_required: caller requires OOB data read to chip->oob_poi
3046 * @page: page number to read
3048 static int nand_read_page_swecc(struct nand_chip *chip, uint8_t *buf,
3049 int oob_required, int page)
3051 struct mtd_info *mtd = nand_to_mtd(chip);
3052 int i, eccsize = chip->ecc.size, ret;
3053 int eccbytes = chip->ecc.bytes;
3054 int eccsteps = chip->ecc.steps;
3056 uint8_t *ecc_calc = chip->ecc.calc_buf;
3057 uint8_t *ecc_code = chip->ecc.code_buf;
3058 unsigned int max_bitflips = 0;
3060 chip->ecc.read_page_raw(chip, buf, 1, page);
3062 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
3063 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3065 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3070 eccsteps = chip->ecc.steps;
3073 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3076 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3078 mtd->ecc_stats.failed++;
3080 mtd->ecc_stats.corrected += stat;
3081 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3084 return max_bitflips;
3088 * nand_read_subpage - [REPLACEABLE] ECC based sub-page read function
3089 * @chip: nand chip info structure
3090 * @data_offs: offset of requested data within the page
3091 * @readlen: data length
3092 * @bufpoi: buffer to store read data
3093 * @page: page number to read
3095 static int nand_read_subpage(struct nand_chip *chip, uint32_t data_offs,
3096 uint32_t readlen, uint8_t *bufpoi, int page)
3098 struct mtd_info *mtd = nand_to_mtd(chip);
3099 int start_step, end_step, num_steps, ret;
3101 int data_col_addr, i, gaps = 0;
3102 int datafrag_len, eccfrag_len, aligned_len, aligned_pos;
3103 int busw = (chip->options & NAND_BUSWIDTH_16) ? 2 : 1;
3104 int index, section = 0;
3105 unsigned int max_bitflips = 0;
3106 struct mtd_oob_region oobregion = { };
3108 /* Column address within the page aligned to ECC size (256bytes) */
3109 start_step = data_offs / chip->ecc.size;
3110 end_step = (data_offs + readlen - 1) / chip->ecc.size;
3111 num_steps = end_step - start_step + 1;
3112 index = start_step * chip->ecc.bytes;
3114 /* Data size aligned to ECC ecc.size */
3115 datafrag_len = num_steps * chip->ecc.size;
3116 eccfrag_len = num_steps * chip->ecc.bytes;
3118 data_col_addr = start_step * chip->ecc.size;
3119 /* If we read not a page aligned data */
3120 p = bufpoi + data_col_addr;
3121 ret = nand_read_page_op(chip, page, data_col_addr, p, datafrag_len);
3126 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
3127 chip->ecc.calculate(chip, p, &chip->ecc.calc_buf[i]);
3130 * The performance is faster if we position offsets according to
3131 * ecc.pos. Let's make sure that there are no gaps in ECC positions.
3133 ret = mtd_ooblayout_find_eccregion(mtd, index, §ion, &oobregion);
3137 if (oobregion.length < eccfrag_len)
3141 ret = nand_change_read_column_op(chip, mtd->writesize,
3142 chip->oob_poi, mtd->oobsize,
3148 * Send the command to read the particular ECC bytes take care
3149 * about buswidth alignment in read_buf.
3151 aligned_pos = oobregion.offset & ~(busw - 1);
3152 aligned_len = eccfrag_len;
3153 if (oobregion.offset & (busw - 1))
3155 if ((oobregion.offset + (num_steps * chip->ecc.bytes)) &
3159 ret = nand_change_read_column_op(chip,
3160 mtd->writesize + aligned_pos,
3161 &chip->oob_poi[aligned_pos],
3162 aligned_len, false);
3167 ret = mtd_ooblayout_get_eccbytes(mtd, chip->ecc.code_buf,
3168 chip->oob_poi, index, eccfrag_len);
3172 p = bufpoi + data_col_addr;
3173 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
3176 stat = chip->ecc.correct(chip, p, &chip->ecc.code_buf[i],
3177 &chip->ecc.calc_buf[i]);
3178 if (stat == -EBADMSG &&
3179 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3180 /* check for empty pages with bitflips */
3181 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3182 &chip->ecc.code_buf[i],
3185 chip->ecc.strength);
3189 mtd->ecc_stats.failed++;
3191 mtd->ecc_stats.corrected += stat;
3192 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3195 return max_bitflips;
3199 * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function
3200 * @chip: nand chip info structure
3201 * @buf: buffer to store read data
3202 * @oob_required: caller requires OOB data read to chip->oob_poi
3203 * @page: page number to read
3205 * Not for syndrome calculating ECC controllers which need a special oob layout.
3207 static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
3208 int oob_required, int page)
3210 struct mtd_info *mtd = nand_to_mtd(chip);
3211 int i, eccsize = chip->ecc.size, ret;
3212 int eccbytes = chip->ecc.bytes;
3213 int eccsteps = chip->ecc.steps;
3215 uint8_t *ecc_calc = chip->ecc.calc_buf;
3216 uint8_t *ecc_code = chip->ecc.code_buf;
3217 unsigned int max_bitflips = 0;
3219 ret = nand_read_page_op(chip, page, 0, NULL, 0);
3223 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3224 chip->ecc.hwctl(chip, NAND_ECC_READ);
3226 ret = nand_read_data_op(chip, p, eccsize, false, false);
3230 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3233 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
3238 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3243 eccsteps = chip->ecc.steps;
3246 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3249 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3250 if (stat == -EBADMSG &&
3251 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3252 /* check for empty pages with bitflips */
3253 stat = nand_check_erased_ecc_chunk(p, eccsize,
3254 &ecc_code[i], eccbytes,
3256 chip->ecc.strength);
3260 mtd->ecc_stats.failed++;
3262 mtd->ecc_stats.corrected += stat;
3263 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3266 return max_bitflips;
3270 * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC
3271 * data read from OOB area
3272 * @chip: nand chip info structure
3273 * @buf: buffer to store read data
3274 * @oob_required: caller requires OOB data read to chip->oob_poi
3275 * @page: page number to read
3277 * Hardware ECC for large page chips, which requires the ECC data to be
3278 * extracted from the OOB before the actual data is read.
3280 int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf,
3281 int oob_required, int page)
3283 struct mtd_info *mtd = nand_to_mtd(chip);
3284 int i, eccsize = chip->ecc.size, ret;
3285 int eccbytes = chip->ecc.bytes;
3286 int eccsteps = chip->ecc.steps;
3288 uint8_t *ecc_code = chip->ecc.code_buf;
3289 unsigned int max_bitflips = 0;
3291 /* Read the OOB area first */
3292 ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3296 /* Move read cursor to start of page */
3297 ret = nand_change_read_column_op(chip, 0, NULL, 0, false);
3301 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3306 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3309 chip->ecc.hwctl(chip, NAND_ECC_READ);
3311 ret = nand_read_data_op(chip, p, eccsize, false, false);
3315 stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
3316 if (stat == -EBADMSG &&
3317 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3318 /* check for empty pages with bitflips */
3319 stat = nand_check_erased_ecc_chunk(p, eccsize,
3322 chip->ecc.strength);
3326 mtd->ecc_stats.failed++;
3328 mtd->ecc_stats.corrected += stat;
3329 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3332 return max_bitflips;
3334 EXPORT_SYMBOL_GPL(nand_read_page_hwecc_oob_first);
3337 * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
3338 * @chip: nand chip info structure
3339 * @buf: buffer to store read data
3340 * @oob_required: caller requires OOB data read to chip->oob_poi
3341 * @page: page number to read
3343 * The hw generator calculates the error syndrome automatically. Therefore we
3344 * need a special oob layout and handling.
3346 static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
3347 int oob_required, int page)
3349 struct mtd_info *mtd = nand_to_mtd(chip);
3350 int ret, i, eccsize = chip->ecc.size;
3351 int eccbytes = chip->ecc.bytes;
3352 int eccsteps = chip->ecc.steps;
3353 int eccpadbytes = eccbytes + chip->ecc.prepad + chip->ecc.postpad;
3355 uint8_t *oob = chip->oob_poi;
3356 unsigned int max_bitflips = 0;
3358 ret = nand_read_page_op(chip, page, 0, NULL, 0);
3362 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3365 chip->ecc.hwctl(chip, NAND_ECC_READ);
3367 ret = nand_read_data_op(chip, p, eccsize, false, false);
3371 if (chip->ecc.prepad) {
3372 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3377 oob += chip->ecc.prepad;
3380 chip->ecc.hwctl(chip, NAND_ECC_READSYN);
3382 ret = nand_read_data_op(chip, oob, eccbytes, false, false);
3386 stat = chip->ecc.correct(chip, p, oob, NULL);
3390 if (chip->ecc.postpad) {
3391 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3396 oob += chip->ecc.postpad;
3399 if (stat == -EBADMSG &&
3400 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3401 /* check for empty pages with bitflips */
3402 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3406 chip->ecc.strength);
3410 mtd->ecc_stats.failed++;
3412 mtd->ecc_stats.corrected += stat;
3413 max_bitflips = max_t(unsigned int, max_bitflips, stat);
3417 /* Calculate remaining oob bytes */
3418 i = mtd->oobsize - (oob - chip->oob_poi);
3420 ret = nand_read_data_op(chip, oob, i, false, false);
3425 return max_bitflips;
3429 * nand_transfer_oob - [INTERN] Transfer oob to client buffer
3430 * @chip: NAND chip object
3431 * @oob: oob destination address
3432 * @ops: oob ops structure
3433 * @len: size of oob to transfer
3435 static uint8_t *nand_transfer_oob(struct nand_chip *chip, uint8_t *oob,
3436 struct mtd_oob_ops *ops, size_t len)
3438 struct mtd_info *mtd = nand_to_mtd(chip);
3441 switch (ops->mode) {
3443 case MTD_OPS_PLACE_OOB:
3445 memcpy(oob, chip->oob_poi + ops->ooboffs, len);
3448 case MTD_OPS_AUTO_OOB:
3449 ret = mtd_ooblayout_get_databytes(mtd, oob, chip->oob_poi,
3460 static void rawnand_enable_cont_reads(struct nand_chip *chip, unsigned int page,
3461 u32 readlen, int col)
3463 struct mtd_info *mtd = nand_to_mtd(chip);
3465 if (!chip->controller->supported_op.cont_read)
3468 if ((col && col + readlen < (3 * mtd->writesize)) ||
3469 (!col && readlen < (2 * mtd->writesize))) {
3470 chip->cont_read.ongoing = false;
3474 chip->cont_read.ongoing = true;
3475 chip->cont_read.first_page = page;
3477 chip->cont_read.first_page++;
3478 chip->cont_read.last_page = page + ((readlen >> chip->page_shift) & chip->pagemask);
3479 rawnand_cap_cont_reads(chip);
3482 static void rawnand_cont_read_skip_first_page(struct nand_chip *chip, unsigned int page)
3484 if (!chip->cont_read.ongoing || page != chip->cont_read.first_page)
3487 chip->cont_read.first_page++;
3488 if (chip->cont_read.first_page == chip->cont_read.pause_page)
3489 chip->cont_read.first_page++;
3490 if (chip->cont_read.first_page >= chip->cont_read.last_page)
3491 chip->cont_read.ongoing = false;
3495 * nand_setup_read_retry - [INTERN] Set the READ RETRY mode
3496 * @chip: NAND chip object
3497 * @retry_mode: the retry mode to use
3499 * Some vendors supply a special command to shift the Vt threshold, to be used
3500 * when there are too many bitflips in a page (i.e., ECC error). After setting
3501 * a new threshold, the host should retry reading the page.
3503 static int nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
3505 pr_debug("setting READ RETRY mode %d\n", retry_mode);
3507 if (retry_mode >= chip->read_retries)
3510 if (!chip->ops.setup_read_retry)
3513 return chip->ops.setup_read_retry(chip, retry_mode);
3516 static void nand_wait_readrdy(struct nand_chip *chip)
3518 const struct nand_interface_config *conf;
3520 if (!(chip->options & NAND_NEED_READRDY))
3523 conf = nand_get_interface_config(chip);
3524 WARN_ON(nand_wait_rdy_op(chip, NAND_COMMON_TIMING_MS(conf, tR_max), 0));
3528 * nand_do_read_ops - [INTERN] Read data with ECC
3529 * @chip: NAND chip object
3530 * @from: offset to read from
3531 * @ops: oob ops structure
3533 * Internal function. Called with chip held.
3535 static int nand_do_read_ops(struct nand_chip *chip, loff_t from,
3536 struct mtd_oob_ops *ops)
3538 int chipnr, page, realpage, col, bytes, aligned, oob_required;
3539 struct mtd_info *mtd = nand_to_mtd(chip);
3541 uint32_t readlen = ops->len;
3542 uint32_t oobreadlen = ops->ooblen;
3543 uint32_t max_oobsize = mtd_oobavail(mtd, ops);
3545 uint8_t *bufpoi, *oob, *buf;
3547 unsigned int max_bitflips = 0;
3549 bool ecc_fail = false;
3551 /* Check if the region is secured */
3552 if (nand_region_is_secured(chip, from, readlen))
3555 chipnr = (int)(from >> chip->chip_shift);
3556 nand_select_target(chip, chipnr);
3558 realpage = (int)(from >> chip->page_shift);
3559 page = realpage & chip->pagemask;
3561 col = (int)(from & (mtd->writesize - 1));
3565 oob_required = oob ? 1 : 0;
3567 rawnand_enable_cont_reads(chip, page, readlen, col);
3570 struct mtd_ecc_stats ecc_stats = mtd->ecc_stats;
3572 bytes = min(mtd->writesize - col, readlen);
3573 aligned = (bytes == mtd->writesize);
3577 else if (chip->options & NAND_USES_DMA)
3578 use_bounce_buf = !virt_addr_valid(buf) ||
3579 !IS_ALIGNED((unsigned long)buf,
3584 /* Is the current page in the buffer? */
3585 if (realpage != chip->pagecache.page || oob) {
3586 bufpoi = use_bounce_buf ? chip->data_buf : buf;
3588 if (use_bounce_buf && aligned)
3589 pr_debug("%s: using read bounce buffer for buf@%p\n",
3594 * Now read the page into the buffer. Absent an error,
3595 * the read methods return max bitflips per ecc step.
3597 if (unlikely(ops->mode == MTD_OPS_RAW))
3598 ret = chip->ecc.read_page_raw(chip, bufpoi,
3601 else if (!aligned && NAND_HAS_SUBPAGE_READ(chip) &&
3603 ret = chip->ecc.read_subpage(chip, col, bytes,
3606 ret = chip->ecc.read_page(chip, bufpoi,
3607 oob_required, page);
3610 /* Invalidate page cache */
3611 chip->pagecache.page = -1;
3616 * Copy back the data in the initial buffer when reading
3617 * partial pages or when a bounce buffer is required.
3619 if (use_bounce_buf) {
3620 if (!NAND_HAS_SUBPAGE_READ(chip) && !oob &&
3621 !(mtd->ecc_stats.failed - ecc_stats.failed) &&
3622 (ops->mode != MTD_OPS_RAW)) {
3623 chip->pagecache.page = realpage;
3624 chip->pagecache.bitflips = ret;
3626 /* Invalidate page cache */
3627 chip->pagecache.page = -1;
3629 memcpy(buf, bufpoi + col, bytes);
3632 if (unlikely(oob)) {
3633 int toread = min(oobreadlen, max_oobsize);
3636 oob = nand_transfer_oob(chip, oob, ops,
3638 oobreadlen -= toread;
3642 nand_wait_readrdy(chip);
3644 if (mtd->ecc_stats.failed - ecc_stats.failed) {
3645 if (retry_mode + 1 < chip->read_retries) {
3647 ret = nand_setup_read_retry(chip,
3652 /* Reset ecc_stats; retry */
3653 mtd->ecc_stats = ecc_stats;
3656 /* No more retry modes; real failure */
3662 max_bitflips = max_t(unsigned int, max_bitflips, ret);
3664 memcpy(buf, chip->data_buf + col, bytes);
3666 max_bitflips = max_t(unsigned int, max_bitflips,
3667 chip->pagecache.bitflips);
3669 rawnand_cont_read_skip_first_page(chip, page);
3674 /* Reset to retry mode 0 */
3676 ret = nand_setup_read_retry(chip, 0);
3685 /* For subsequent reads align to page boundary */
3687 /* Increment page address */
3690 page = realpage & chip->pagemask;
3691 /* Check, if we cross a chip boundary */
3694 nand_deselect_target(chip);
3695 nand_select_target(chip, chipnr);
3698 nand_deselect_target(chip);
3700 ops->retlen = ops->len - (size_t) readlen;
3702 ops->oobretlen = ops->ooblen - oobreadlen;
3710 return max_bitflips;
3714 * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
3715 * @chip: nand chip info structure
3716 * @page: page number to read
3718 int nand_read_oob_std(struct nand_chip *chip, int page)
3720 struct mtd_info *mtd = nand_to_mtd(chip);
3722 return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3724 EXPORT_SYMBOL(nand_read_oob_std);
3727 * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC
3729 * @chip: nand chip info structure
3730 * @page: page number to read
3732 static int nand_read_oob_syndrome(struct nand_chip *chip, int page)
3734 struct mtd_info *mtd = nand_to_mtd(chip);
3735 int length = mtd->oobsize;
3736 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3737 int eccsize = chip->ecc.size;
3738 uint8_t *bufpoi = chip->oob_poi;
3739 int i, toread, sndrnd = 0, pos, ret;
3741 ret = nand_read_page_op(chip, page, chip->ecc.size, NULL, 0);
3745 for (i = 0; i < chip->ecc.steps; i++) {
3749 pos = eccsize + i * (eccsize + chunk);
3750 if (mtd->writesize > 512)
3751 ret = nand_change_read_column_op(chip, pos,
3755 ret = nand_read_page_op(chip, page, pos, NULL,
3762 toread = min_t(int, length, chunk);
3764 ret = nand_read_data_op(chip, bufpoi, toread, false, false);
3772 ret = nand_read_data_op(chip, bufpoi, length, false, false);
3781 * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function
3782 * @chip: nand chip info structure
3783 * @page: page number to write
3785 int nand_write_oob_std(struct nand_chip *chip, int page)
3787 struct mtd_info *mtd = nand_to_mtd(chip);
3789 return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
3792 EXPORT_SYMBOL(nand_write_oob_std);
3795 * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC
3796 * with syndrome - only for large page flash
3797 * @chip: nand chip info structure
3798 * @page: page number to write
3800 static int nand_write_oob_syndrome(struct nand_chip *chip, int page)
3802 struct mtd_info *mtd = nand_to_mtd(chip);
3803 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3804 int eccsize = chip->ecc.size, length = mtd->oobsize;
3805 int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps;
3806 const uint8_t *bufpoi = chip->oob_poi;
3809 * data-ecc-data-ecc ... ecc-oob
3811 * data-pad-ecc-pad-data-pad .... ecc-pad-oob
3813 if (!chip->ecc.prepad && !chip->ecc.postpad) {
3814 pos = steps * (eccsize + chunk);
3819 ret = nand_prog_page_begin_op(chip, page, pos, NULL, 0);
3823 for (i = 0; i < steps; i++) {
3825 if (mtd->writesize <= 512) {
3826 uint32_t fill = 0xFFFFFFFF;
3830 int num = min_t(int, len, 4);
3832 ret = nand_write_data_op(chip, &fill,
3840 pos = eccsize + i * (eccsize + chunk);
3841 ret = nand_change_write_column_op(chip, pos,
3849 len = min_t(int, length, chunk);
3851 ret = nand_write_data_op(chip, bufpoi, len, false);
3859 ret = nand_write_data_op(chip, bufpoi, length, false);
3864 return nand_prog_page_end_op(chip);
3868 * nand_do_read_oob - [INTERN] NAND read out-of-band
3869 * @chip: NAND chip object
3870 * @from: offset to read from
3871 * @ops: oob operations description structure
3873 * NAND read out-of-band data from the spare area.
3875 static int nand_do_read_oob(struct nand_chip *chip, loff_t from,
3876 struct mtd_oob_ops *ops)
3878 struct mtd_info *mtd = nand_to_mtd(chip);
3879 unsigned int max_bitflips = 0;
3880 int page, realpage, chipnr;
3881 struct mtd_ecc_stats stats;
3882 int readlen = ops->ooblen;
3884 uint8_t *buf = ops->oobbuf;
3887 pr_debug("%s: from = 0x%08Lx, len = %i\n",
3888 __func__, (unsigned long long)from, readlen);
3890 /* Check if the region is secured */
3891 if (nand_region_is_secured(chip, from, readlen))
3894 stats = mtd->ecc_stats;
3896 len = mtd_oobavail(mtd, ops);
3898 chipnr = (int)(from >> chip->chip_shift);
3899 nand_select_target(chip, chipnr);
3901 /* Shift to get page */
3902 realpage = (int)(from >> chip->page_shift);
3903 page = realpage & chip->pagemask;
3906 if (ops->mode == MTD_OPS_RAW)
3907 ret = chip->ecc.read_oob_raw(chip, page);
3909 ret = chip->ecc.read_oob(chip, page);
3914 len = min(len, readlen);
3915 buf = nand_transfer_oob(chip, buf, ops, len);
3917 nand_wait_readrdy(chip);
3919 max_bitflips = max_t(unsigned int, max_bitflips, ret);
3925 /* Increment page address */
3928 page = realpage & chip->pagemask;
3929 /* Check, if we cross a chip boundary */
3932 nand_deselect_target(chip);
3933 nand_select_target(chip, chipnr);
3936 nand_deselect_target(chip);
3938 ops->oobretlen = ops->ooblen - readlen;
3943 if (mtd->ecc_stats.failed - stats.failed)
3946 return max_bitflips;
3950 * nand_read_oob - [MTD Interface] NAND read data and/or out-of-band
3951 * @mtd: MTD device structure
3952 * @from: offset to read from
3953 * @ops: oob operation description structure
3955 * NAND read data and/or out-of-band data.
3957 static int nand_read_oob(struct mtd_info *mtd, loff_t from,
3958 struct mtd_oob_ops *ops)
3960 struct nand_chip *chip = mtd_to_nand(mtd);
3961 struct mtd_ecc_stats old_stats;
3966 if (ops->mode != MTD_OPS_PLACE_OOB &&
3967 ops->mode != MTD_OPS_AUTO_OOB &&
3968 ops->mode != MTD_OPS_RAW)
3971 nand_get_device(chip);
3973 old_stats = mtd->ecc_stats;
3976 ret = nand_do_read_oob(chip, from, ops);
3978 ret = nand_do_read_ops(chip, from, ops);
3981 ops->stats->uncorrectable_errors +=
3982 mtd->ecc_stats.failed - old_stats.failed;
3983 ops->stats->corrected_bitflips +=
3984 mtd->ecc_stats.corrected - old_stats.corrected;
3987 nand_release_device(chip);
3992 * nand_write_page_raw_notsupp - dummy raw page write function
3993 * @chip: nand chip info structure
3995 * @oob_required: must write chip->oob_poi to OOB
3996 * @page: page number to write
3998 * Returns -ENOTSUPP unconditionally.
4000 int nand_write_page_raw_notsupp(struct nand_chip *chip, const u8 *buf,
4001 int oob_required, int page)
4007 * nand_write_page_raw - [INTERN] raw page write function
4008 * @chip: nand chip info structure
4010 * @oob_required: must write chip->oob_poi to OOB
4011 * @page: page number to write
4013 * Not for syndrome calculating ECC controllers, which use a special oob layout.
4015 int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
4016 int oob_required, int page)
4018 struct mtd_info *mtd = nand_to_mtd(chip);
4021 ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
4026 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize,
4032 return nand_prog_page_end_op(chip);
4034 EXPORT_SYMBOL(nand_write_page_raw);
4037 * nand_monolithic_write_page_raw - Monolithic page write in raw mode
4038 * @chip: NAND chip info structure
4039 * @buf: data buffer to write
4040 * @oob_required: must write chip->oob_poi to OOB
4041 * @page: page number to write
4043 * This is a raw page write, ie. without any error detection/correction.
4044 * Monolithic means we are requesting all the relevant data (main plus
4045 * eventually OOB) to be sent over the bus and effectively programmed
4046 * into the NAND chip arrays in a single operation. This is an
4047 * alternative to nand_write_page_raw(), which first sends the main
4048 * data, then eventually send the OOB data by latching more data
4049 * cycles on the NAND bus, and finally sends the program command to
4050 * synchronyze the NAND chip cache.
4052 int nand_monolithic_write_page_raw(struct nand_chip *chip, const u8 *buf,
4053 int oob_required, int page)
4055 struct mtd_info *mtd = nand_to_mtd(chip);
4056 unsigned int size = mtd->writesize;
4057 u8 *write_buf = (u8 *)buf;
4060 size += mtd->oobsize;
4062 if (buf != chip->data_buf) {
4063 write_buf = nand_get_data_buf(chip);
4064 memcpy(write_buf, buf, mtd->writesize);
4068 return nand_prog_page_op(chip, page, 0, write_buf, size);
4070 EXPORT_SYMBOL(nand_monolithic_write_page_raw);
4073 * nand_write_page_raw_syndrome - [INTERN] raw page write function
4074 * @chip: nand chip info structure
4076 * @oob_required: must write chip->oob_poi to OOB
4077 * @page: page number to write
4079 * We need a special oob layout and handling even when ECC isn't checked.
4081 static int nand_write_page_raw_syndrome(struct nand_chip *chip,
4082 const uint8_t *buf, int oob_required,
4085 struct mtd_info *mtd = nand_to_mtd(chip);
4086 int eccsize = chip->ecc.size;
4087 int eccbytes = chip->ecc.bytes;
4088 uint8_t *oob = chip->oob_poi;
4089 int steps, size, ret;
4091 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4095 for (steps = chip->ecc.steps; steps > 0; steps--) {
4096 ret = nand_write_data_op(chip, buf, eccsize, false);
4102 if (chip->ecc.prepad) {
4103 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4108 oob += chip->ecc.prepad;
4111 ret = nand_write_data_op(chip, oob, eccbytes, false);
4117 if (chip->ecc.postpad) {
4118 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4123 oob += chip->ecc.postpad;
4127 size = mtd->oobsize - (oob - chip->oob_poi);
4129 ret = nand_write_data_op(chip, oob, size, false);
4134 return nand_prog_page_end_op(chip);
4137 * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
4138 * @chip: nand chip info structure
4140 * @oob_required: must write chip->oob_poi to OOB
4141 * @page: page number to write
4143 static int nand_write_page_swecc(struct nand_chip *chip, const uint8_t *buf,
4144 int oob_required, int page)
4146 struct mtd_info *mtd = nand_to_mtd(chip);
4147 int i, eccsize = chip->ecc.size, ret;
4148 int eccbytes = chip->ecc.bytes;
4149 int eccsteps = chip->ecc.steps;
4150 uint8_t *ecc_calc = chip->ecc.calc_buf;
4151 const uint8_t *p = buf;
4153 /* Software ECC calculation */
4154 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
4155 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4157 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4162 return chip->ecc.write_page_raw(chip, buf, 1, page);
4166 * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function
4167 * @chip: nand chip info structure
4169 * @oob_required: must write chip->oob_poi to OOB
4170 * @page: page number to write
4172 static int nand_write_page_hwecc(struct nand_chip *chip, const uint8_t *buf,
4173 int oob_required, int page)
4175 struct mtd_info *mtd = nand_to_mtd(chip);
4176 int i, eccsize = chip->ecc.size, ret;
4177 int eccbytes = chip->ecc.bytes;
4178 int eccsteps = chip->ecc.steps;
4179 uint8_t *ecc_calc = chip->ecc.calc_buf;
4180 const uint8_t *p = buf;
4182 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4186 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4187 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4189 ret = nand_write_data_op(chip, p, eccsize, false);
4193 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4196 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4201 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4205 return nand_prog_page_end_op(chip);
4210 * nand_write_subpage_hwecc - [REPLACEABLE] hardware ECC based subpage write
4211 * @chip: nand chip info structure
4212 * @offset: column address of subpage within the page
4213 * @data_len: data length
4215 * @oob_required: must write chip->oob_poi to OOB
4216 * @page: page number to write
4218 static int nand_write_subpage_hwecc(struct nand_chip *chip, uint32_t offset,
4219 uint32_t data_len, const uint8_t *buf,
4220 int oob_required, int page)
4222 struct mtd_info *mtd = nand_to_mtd(chip);
4223 uint8_t *oob_buf = chip->oob_poi;
4224 uint8_t *ecc_calc = chip->ecc.calc_buf;
4225 int ecc_size = chip->ecc.size;
4226 int ecc_bytes = chip->ecc.bytes;
4227 int ecc_steps = chip->ecc.steps;
4228 uint32_t start_step = offset / ecc_size;
4229 uint32_t end_step = (offset + data_len - 1) / ecc_size;
4230 int oob_bytes = mtd->oobsize / ecc_steps;
4233 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4237 for (step = 0; step < ecc_steps; step++) {
4238 /* configure controller for WRITE access */
4239 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4241 /* write data (untouched subpages already masked by 0xFF) */
4242 ret = nand_write_data_op(chip, buf, ecc_size, false);
4246 /* mask ECC of un-touched subpages by padding 0xFF */
4247 if ((step < start_step) || (step > end_step))
4248 memset(ecc_calc, 0xff, ecc_bytes);
4250 chip->ecc.calculate(chip, buf, ecc_calc);
4252 /* mask OOB of un-touched subpages by padding 0xFF */
4253 /* if oob_required, preserve OOB metadata of written subpage */
4254 if (!oob_required || (step < start_step) || (step > end_step))
4255 memset(oob_buf, 0xff, oob_bytes);
4258 ecc_calc += ecc_bytes;
4259 oob_buf += oob_bytes;
4262 /* copy calculated ECC for whole page to chip->buffer->oob */
4263 /* this include masked-value(0xFF) for unwritten subpages */
4264 ecc_calc = chip->ecc.calc_buf;
4265 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4270 /* write OOB buffer to NAND device */
4271 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4275 return nand_prog_page_end_op(chip);
4280 * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write
4281 * @chip: nand chip info structure
4283 * @oob_required: must write chip->oob_poi to OOB
4284 * @page: page number to write
4286 * The hw generator calculates the error syndrome automatically. Therefore we
4287 * need a special oob layout and handling.
4289 static int nand_write_page_syndrome(struct nand_chip *chip, const uint8_t *buf,
4290 int oob_required, int page)
4292 struct mtd_info *mtd = nand_to_mtd(chip);
4293 int i, eccsize = chip->ecc.size;
4294 int eccbytes = chip->ecc.bytes;
4295 int eccsteps = chip->ecc.steps;
4296 const uint8_t *p = buf;
4297 uint8_t *oob = chip->oob_poi;
4300 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
4304 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4305 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4307 ret = nand_write_data_op(chip, p, eccsize, false);
4311 if (chip->ecc.prepad) {
4312 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4317 oob += chip->ecc.prepad;
4320 chip->ecc.calculate(chip, p, oob);
4322 ret = nand_write_data_op(chip, oob, eccbytes, false);
4328 if (chip->ecc.postpad) {
4329 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4334 oob += chip->ecc.postpad;
4338 /* Calculate remaining oob bytes */
4339 i = mtd->oobsize - (oob - chip->oob_poi);
4341 ret = nand_write_data_op(chip, oob, i, false);
4346 return nand_prog_page_end_op(chip);
4350 * nand_write_page - write one page
4351 * @chip: NAND chip descriptor
4352 * @offset: address offset within the page
4353 * @data_len: length of actual data to be written
4354 * @buf: the data to write
4355 * @oob_required: must write chip->oob_poi to OOB
4356 * @page: page number to write
4357 * @raw: use _raw version of write_page
4359 static int nand_write_page(struct nand_chip *chip, uint32_t offset,
4360 int data_len, const uint8_t *buf, int oob_required,
4363 struct mtd_info *mtd = nand_to_mtd(chip);
4364 int status, subpage;
4366 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) &&
4367 chip->ecc.write_subpage)
4368 subpage = offset || (data_len < mtd->writesize);
4373 status = chip->ecc.write_page_raw(chip, buf, oob_required,
4376 status = chip->ecc.write_subpage(chip, offset, data_len, buf,
4377 oob_required, page);
4379 status = chip->ecc.write_page(chip, buf, oob_required, page);
4387 #define NOTALIGNED(x) ((x & (chip->subpagesize - 1)) != 0)
4390 * nand_do_write_ops - [INTERN] NAND write with ECC
4391 * @chip: NAND chip object
4392 * @to: offset to write to
4393 * @ops: oob operations description structure
4395 * NAND write with ECC.
4397 static int nand_do_write_ops(struct nand_chip *chip, loff_t to,
4398 struct mtd_oob_ops *ops)
4400 struct mtd_info *mtd = nand_to_mtd(chip);
4401 int chipnr, realpage, page, column;
4402 uint32_t writelen = ops->len;
4404 uint32_t oobwritelen = ops->ooblen;
4405 uint32_t oobmaxlen = mtd_oobavail(mtd, ops);
4407 uint8_t *oob = ops->oobbuf;
4408 uint8_t *buf = ops->datbuf;
4410 int oob_required = oob ? 1 : 0;
4416 /* Reject writes, which are not page aligned */
4417 if (NOTALIGNED(to) || NOTALIGNED(ops->len)) {
4418 pr_notice("%s: attempt to write non page aligned data\n",
4423 /* Check if the region is secured */
4424 if (nand_region_is_secured(chip, to, writelen))
4427 column = to & (mtd->writesize - 1);
4429 chipnr = (int)(to >> chip->chip_shift);
4430 nand_select_target(chip, chipnr);
4432 /* Check, if it is write protected */
4433 if (nand_check_wp(chip)) {
4438 realpage = (int)(to >> chip->page_shift);
4439 page = realpage & chip->pagemask;
4441 /* Invalidate the page cache, when we write to the cached page */
4442 if (to <= ((loff_t)chip->pagecache.page << chip->page_shift) &&
4443 ((loff_t)chip->pagecache.page << chip->page_shift) < (to + ops->len))
4444 chip->pagecache.page = -1;
4446 /* Don't allow multipage oob writes with offset */
4447 if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen)) {
4453 int bytes = mtd->writesize;
4454 uint8_t *wbuf = buf;
4456 int part_pagewr = (column || writelen < mtd->writesize);
4460 else if (chip->options & NAND_USES_DMA)
4461 use_bounce_buf = !virt_addr_valid(buf) ||
4462 !IS_ALIGNED((unsigned long)buf,
4468 * Copy the data from the initial buffer when doing partial page
4469 * writes or when a bounce buffer is required.
4471 if (use_bounce_buf) {
4472 pr_debug("%s: using write bounce buffer for buf@%p\n",
4475 bytes = min_t(int, bytes - column, writelen);
4476 wbuf = nand_get_data_buf(chip);
4477 memset(wbuf, 0xff, mtd->writesize);
4478 memcpy(&wbuf[column], buf, bytes);
4481 if (unlikely(oob)) {
4482 size_t len = min(oobwritelen, oobmaxlen);
4483 oob = nand_fill_oob(chip, oob, len, ops);
4486 /* We still need to erase leftover OOB data */
4487 memset(chip->oob_poi, 0xff, mtd->oobsize);
4490 ret = nand_write_page(chip, column, bytes, wbuf,
4492 (ops->mode == MTD_OPS_RAW));
4504 page = realpage & chip->pagemask;
4505 /* Check, if we cross a chip boundary */
4508 nand_deselect_target(chip);
4509 nand_select_target(chip, chipnr);
4513 ops->retlen = ops->len - writelen;
4515 ops->oobretlen = ops->ooblen;
4518 nand_deselect_target(chip);
4523 * panic_nand_write - [MTD Interface] NAND write with ECC
4524 * @mtd: MTD device structure
4525 * @to: offset to write to
4526 * @len: number of bytes to write
4527 * @retlen: pointer to variable to store the number of written bytes
4528 * @buf: the data to write
4530 * NAND write with ECC. Used when performing writes in interrupt context, this
4531 * may for example be called by mtdoops when writing an oops while in panic.
4533 static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
4534 size_t *retlen, const uint8_t *buf)
4536 struct nand_chip *chip = mtd_to_nand(mtd);
4537 int chipnr = (int)(to >> chip->chip_shift);
4538 struct mtd_oob_ops ops;
4541 nand_select_target(chip, chipnr);
4543 /* Wait for the device to get ready */
4544 panic_nand_wait(chip, 400);
4546 memset(&ops, 0, sizeof(ops));
4548 ops.datbuf = (uint8_t *)buf;
4549 ops.mode = MTD_OPS_PLACE_OOB;
4551 ret = nand_do_write_ops(chip, to, &ops);
4553 *retlen = ops.retlen;
4558 * nand_write_oob - [MTD Interface] NAND write data and/or out-of-band
4559 * @mtd: MTD device structure
4560 * @to: offset to write to
4561 * @ops: oob operation description structure
4563 static int nand_write_oob(struct mtd_info *mtd, loff_t to,
4564 struct mtd_oob_ops *ops)
4566 struct nand_chip *chip = mtd_to_nand(mtd);
4571 nand_get_device(chip);
4573 switch (ops->mode) {
4574 case MTD_OPS_PLACE_OOB:
4575 case MTD_OPS_AUTO_OOB:
4584 ret = nand_do_write_oob(chip, to, ops);
4586 ret = nand_do_write_ops(chip, to, ops);
4589 nand_release_device(chip);
4594 * nand_erase - [MTD Interface] erase block(s)
4595 * @mtd: MTD device structure
4596 * @instr: erase instruction
4598 * Erase one ore more blocks.
4600 static int nand_erase(struct mtd_info *mtd, struct erase_info *instr)
4602 return nand_erase_nand(mtd_to_nand(mtd), instr, 0);
4606 * nand_erase_nand - [INTERN] erase block(s)
4607 * @chip: NAND chip object
4608 * @instr: erase instruction
4609 * @allowbbt: allow erasing the bbt area
4611 * Erase one ore more blocks.
4613 int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr,
4616 int page, pages_per_block, ret, chipnr;
4619 pr_debug("%s: start = 0x%012llx, len = %llu\n",
4620 __func__, (unsigned long long)instr->addr,
4621 (unsigned long long)instr->len);
4623 if (check_offs_len(chip, instr->addr, instr->len))
4626 /* Check if the region is secured */
4627 if (nand_region_is_secured(chip, instr->addr, instr->len))
4630 /* Grab the lock and see if the device is available */
4631 nand_get_device(chip);
4633 /* Shift to get first page */
4634 page = (int)(instr->addr >> chip->page_shift);
4635 chipnr = (int)(instr->addr >> chip->chip_shift);
4637 /* Calculate pages in each block */
4638 pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift);
4640 /* Select the NAND device */
4641 nand_select_target(chip, chipnr);
4643 /* Check, if it is write protected */
4644 if (nand_check_wp(chip)) {
4645 pr_debug("%s: device is write protected!\n",
4651 /* Loop through the pages */
4655 loff_t ofs = (loff_t)page << chip->page_shift;
4657 /* Check if we have a bad block, we do not erase bad blocks! */
4658 if (nand_block_checkbad(chip, ((loff_t) page) <<
4659 chip->page_shift, allowbbt)) {
4660 pr_warn("%s: attempt to erase a bad block at 0x%08llx\n",
4661 __func__, (unsigned long long)ofs);
4667 * Invalidate the page cache, if we erase the block which
4668 * contains the current cached page.
4670 if (page <= chip->pagecache.page && chip->pagecache.page <
4671 (page + pages_per_block))
4672 chip->pagecache.page = -1;
4674 ret = nand_erase_op(chip, (page & chip->pagemask) >>
4675 (chip->phys_erase_shift - chip->page_shift));
4677 pr_debug("%s: failed erase, page 0x%08x\n",
4679 instr->fail_addr = ofs;
4683 /* Increment page address and decrement length */
4684 len -= (1ULL << chip->phys_erase_shift);
4685 page += pages_per_block;
4687 /* Check, if we cross a chip boundary */
4688 if (len && !(page & chip->pagemask)) {
4690 nand_deselect_target(chip);
4691 nand_select_target(chip, chipnr);
4698 /* Deselect and wake up anyone waiting on the device */
4699 nand_deselect_target(chip);
4700 nand_release_device(chip);
4702 /* Return more or less happy */
4707 * nand_sync - [MTD Interface] sync
4708 * @mtd: MTD device structure
4710 * Sync is actually a wait for chip ready function.
4712 static void nand_sync(struct mtd_info *mtd)
4714 struct nand_chip *chip = mtd_to_nand(mtd);
4716 pr_debug("%s: called\n", __func__);
4718 /* Grab the lock and see if the device is available */
4719 nand_get_device(chip);
4720 /* Release it and go back */
4721 nand_release_device(chip);
4725 * nand_block_isbad - [MTD Interface] Check if block at offset is bad
4726 * @mtd: MTD device structure
4727 * @offs: offset relative to mtd start
4729 static int nand_block_isbad(struct mtd_info *mtd, loff_t offs)
4731 struct nand_chip *chip = mtd_to_nand(mtd);
4732 int chipnr = (int)(offs >> chip->chip_shift);
4735 /* Select the NAND device */
4736 nand_get_device(chip);
4738 nand_select_target(chip, chipnr);
4740 ret = nand_block_checkbad(chip, offs, 0);
4742 nand_deselect_target(chip);
4743 nand_release_device(chip);
4749 * nand_block_markbad - [MTD Interface] Mark block at the given offset as bad
4750 * @mtd: MTD device structure
4751 * @ofs: offset relative to mtd start
4753 static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
4757 ret = nand_block_isbad(mtd, ofs);
4759 /* If it was bad already, return success and do nothing */
4765 return nand_block_markbad_lowlevel(mtd_to_nand(mtd), ofs);
4769 * nand_suspend - [MTD Interface] Suspend the NAND flash
4770 * @mtd: MTD device structure
4772 * Returns 0 for success or negative error code otherwise.
4774 static int nand_suspend(struct mtd_info *mtd)
4776 struct nand_chip *chip = mtd_to_nand(mtd);
4779 mutex_lock(&chip->lock);
4780 if (chip->ops.suspend)
4781 ret = chip->ops.suspend(chip);
4783 chip->suspended = 1;
4784 mutex_unlock(&chip->lock);
4790 * nand_resume - [MTD Interface] Resume the NAND flash
4791 * @mtd: MTD device structure
4793 static void nand_resume(struct mtd_info *mtd)
4795 struct nand_chip *chip = mtd_to_nand(mtd);
4797 mutex_lock(&chip->lock);
4798 if (chip->suspended) {
4799 if (chip->ops.resume)
4800 chip->ops.resume(chip);
4801 chip->suspended = 0;
4803 pr_err("%s called for a chip which is not in suspended state\n",
4806 mutex_unlock(&chip->lock);
4808 wake_up_all(&chip->resume_wq);
4812 * nand_shutdown - [MTD Interface] Finish the current NAND operation and
4813 * prevent further operations
4814 * @mtd: MTD device structure
4816 static void nand_shutdown(struct mtd_info *mtd)
4822 * nand_lock - [MTD Interface] Lock the NAND flash
4823 * @mtd: MTD device structure
4824 * @ofs: offset byte address
4825 * @len: number of bytes to lock (must be a multiple of block/page size)
4827 static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4829 struct nand_chip *chip = mtd_to_nand(mtd);
4831 if (!chip->ops.lock_area)
4834 return chip->ops.lock_area(chip, ofs, len);
4838 * nand_unlock - [MTD Interface] Unlock the NAND flash
4839 * @mtd: MTD device structure
4840 * @ofs: offset byte address
4841 * @len: number of bytes to unlock (must be a multiple of block/page size)
4843 static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
4845 struct nand_chip *chip = mtd_to_nand(mtd);
4847 if (!chip->ops.unlock_area)
4850 return chip->ops.unlock_area(chip, ofs, len);
4853 /* Set default functions */
4854 static void nand_set_defaults(struct nand_chip *chip)
4856 /* If no controller is provided, use the dummy, legacy one. */
4857 if (!chip->controller) {
4858 chip->controller = &chip->legacy.dummy_controller;
4859 nand_controller_init(chip->controller);
4862 nand_legacy_set_defaults(chip);
4864 if (!chip->buf_align)
4865 chip->buf_align = 1;
4868 /* Sanitize ONFI strings so we can safely print them */
4869 void sanitize_string(uint8_t *s, size_t len)
4873 /* Null terminate */
4876 /* Remove non printable chars */
4877 for (i = 0; i < len - 1; i++) {
4878 if (s[i] < ' ' || s[i] > 127)
4882 /* Remove trailing spaces */
4887 * nand_id_has_period - Check if an ID string has a given wraparound period
4888 * @id_data: the ID string
4889 * @arrlen: the length of the @id_data array
4890 * @period: the period of repitition
4892 * Check if an ID string is repeated within a given sequence of bytes at
4893 * specific repetition interval period (e.g., {0x20,0x01,0x7F,0x20} has a
4894 * period of 3). This is a helper function for nand_id_len(). Returns non-zero
4895 * if the repetition has a period of @period; otherwise, returns zero.
4897 static int nand_id_has_period(u8 *id_data, int arrlen, int period)
4900 for (i = 0; i < period; i++)
4901 for (j = i + period; j < arrlen; j += period)
4902 if (id_data[i] != id_data[j])
4908 * nand_id_len - Get the length of an ID string returned by CMD_READID
4909 * @id_data: the ID string
4910 * @arrlen: the length of the @id_data array
4912 * Returns the length of the ID string, according to known wraparound/trailing
4913 * zero patterns. If no pattern exists, returns the length of the array.
4915 static int nand_id_len(u8 *id_data, int arrlen)
4917 int last_nonzero, period;
4919 /* Find last non-zero byte */
4920 for (last_nonzero = arrlen - 1; last_nonzero >= 0; last_nonzero--)
4921 if (id_data[last_nonzero])
4925 if (last_nonzero < 0)
4928 /* Calculate wraparound period */
4929 for (period = 1; period < arrlen; period++)
4930 if (nand_id_has_period(id_data, arrlen, period))
4933 /* There's a repeated pattern */
4934 if (period < arrlen)
4937 /* There are trailing zeros */
4938 if (last_nonzero < arrlen - 1)
4939 return last_nonzero + 1;
4941 /* No pattern detected */
4945 /* Extract the bits of per cell from the 3rd byte of the extended ID */
4946 static int nand_get_bits_per_cell(u8 cellinfo)
4950 bits = cellinfo & NAND_CI_CELLTYPE_MSK;
4951 bits >>= NAND_CI_CELLTYPE_SHIFT;
4956 * Many new NAND share similar device ID codes, which represent the size of the
4957 * chip. The rest of the parameters must be decoded according to generic or
4958 * manufacturer-specific "extended ID" decoding patterns.
4960 void nand_decode_ext_id(struct nand_chip *chip)
4962 struct nand_memory_organization *memorg;
4963 struct mtd_info *mtd = nand_to_mtd(chip);
4965 u8 *id_data = chip->id.data;
4967 memorg = nanddev_get_memorg(&chip->base);
4969 /* The 3rd id byte holds MLC / multichip data */
4970 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
4971 /* The 4th id byte is the important one */
4975 memorg->pagesize = 1024 << (extid & 0x03);
4976 mtd->writesize = memorg->pagesize;
4979 memorg->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9);
4980 mtd->oobsize = memorg->oobsize;
4982 /* Calc blocksize. Blocksize is multiples of 64KiB */
4983 memorg->pages_per_eraseblock = ((64 * 1024) << (extid & 0x03)) /
4985 mtd->erasesize = (64 * 1024) << (extid & 0x03);
4987 /* Get buswidth information */
4989 chip->options |= NAND_BUSWIDTH_16;
4991 EXPORT_SYMBOL_GPL(nand_decode_ext_id);
4994 * Old devices have chip data hardcoded in the device ID table. nand_decode_id
4995 * decodes a matching ID table entry and assigns the MTD size parameters for
4998 static void nand_decode_id(struct nand_chip *chip, struct nand_flash_dev *type)
5000 struct mtd_info *mtd = nand_to_mtd(chip);
5001 struct nand_memory_organization *memorg;
5003 memorg = nanddev_get_memorg(&chip->base);
5005 memorg->pages_per_eraseblock = type->erasesize / type->pagesize;
5006 mtd->erasesize = type->erasesize;
5007 memorg->pagesize = type->pagesize;
5008 mtd->writesize = memorg->pagesize;
5009 memorg->oobsize = memorg->pagesize / 32;
5010 mtd->oobsize = memorg->oobsize;
5012 /* All legacy ID NAND are small-page, SLC */
5013 memorg->bits_per_cell = 1;
5017 * Set the bad block marker/indicator (BBM/BBI) patterns according to some
5018 * heuristic patterns using various detected parameters (e.g., manufacturer,
5019 * page size, cell-type information).
5021 static void nand_decode_bbm_options(struct nand_chip *chip)
5023 struct mtd_info *mtd = nand_to_mtd(chip);
5025 /* Set the bad block position */
5026 if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16))
5027 chip->badblockpos = NAND_BBM_POS_LARGE;
5029 chip->badblockpos = NAND_BBM_POS_SMALL;
5032 static inline bool is_full_id_nand(struct nand_flash_dev *type)
5034 return type->id_len;
5037 static bool find_full_id_nand(struct nand_chip *chip,
5038 struct nand_flash_dev *type)
5040 struct nand_device *base = &chip->base;
5041 struct nand_ecc_props requirements;
5042 struct mtd_info *mtd = nand_to_mtd(chip);
5043 struct nand_memory_organization *memorg;
5044 u8 *id_data = chip->id.data;
5046 memorg = nanddev_get_memorg(&chip->base);
5048 if (!strncmp(type->id, id_data, type->id_len)) {
5049 memorg->pagesize = type->pagesize;
5050 mtd->writesize = memorg->pagesize;
5051 memorg->pages_per_eraseblock = type->erasesize /
5053 mtd->erasesize = type->erasesize;
5054 memorg->oobsize = type->oobsize;
5055 mtd->oobsize = memorg->oobsize;
5057 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
5058 memorg->eraseblocks_per_lun =
5059 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5061 memorg->pages_per_eraseblock);
5062 chip->options |= type->options;
5063 requirements.strength = NAND_ECC_STRENGTH(type);
5064 requirements.step_size = NAND_ECC_STEP(type);
5065 nanddev_set_ecc_requirements(base, &requirements);
5067 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5068 if (!chip->parameters.model)
5077 * Manufacturer detection. Only used when the NAND is not ONFI or JEDEC
5078 * compliant and does not have a full-id or legacy-id entry in the nand_ids
5081 static void nand_manufacturer_detect(struct nand_chip *chip)
5084 * Try manufacturer detection if available and use
5085 * nand_decode_ext_id() otherwise.
5087 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5088 chip->manufacturer.desc->ops->detect) {
5089 struct nand_memory_organization *memorg;
5091 memorg = nanddev_get_memorg(&chip->base);
5093 /* The 3rd id byte holds MLC / multichip data */
5094 memorg->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]);
5095 chip->manufacturer.desc->ops->detect(chip);
5097 nand_decode_ext_id(chip);
5102 * Manufacturer initialization. This function is called for all NANDs including
5103 * ONFI and JEDEC compliant ones.
5104 * Manufacturer drivers should put all their specific initialization code in
5105 * their ->init() hook.
5107 static int nand_manufacturer_init(struct nand_chip *chip)
5109 if (!chip->manufacturer.desc || !chip->manufacturer.desc->ops ||
5110 !chip->manufacturer.desc->ops->init)
5113 return chip->manufacturer.desc->ops->init(chip);
5117 * Manufacturer cleanup. This function is called for all NANDs including
5118 * ONFI and JEDEC compliant ones.
5119 * Manufacturer drivers should put all their specific cleanup code in their
5122 static void nand_manufacturer_cleanup(struct nand_chip *chip)
5124 /* Release manufacturer private data */
5125 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5126 chip->manufacturer.desc->ops->cleanup)
5127 chip->manufacturer.desc->ops->cleanup(chip);
5131 nand_manufacturer_name(const struct nand_manufacturer_desc *manufacturer_desc)
5133 return manufacturer_desc ? manufacturer_desc->name : "Unknown";
5136 static void rawnand_check_data_only_read_support(struct nand_chip *chip)
5138 /* Use an arbitrary size for the check */
5139 if (!nand_read_data_op(chip, NULL, SZ_512, true, true))
5140 chip->controller->supported_op.data_only_read = 1;
5143 static void rawnand_early_check_supported_ops(struct nand_chip *chip)
5145 /* The supported_op fields should not be set by individual drivers */
5146 WARN_ON_ONCE(chip->controller->supported_op.data_only_read);
5148 if (!nand_has_exec_op(chip))
5151 rawnand_check_data_only_read_support(chip);
5154 static void rawnand_check_cont_read_support(struct nand_chip *chip)
5156 struct mtd_info *mtd = nand_to_mtd(chip);
5158 if (!chip->parameters.supports_read_cache)
5161 if (chip->read_retries)
5164 if (!nand_lp_exec_cont_read_page_op(chip, 0, 0, NULL,
5165 mtd->writesize, true))
5166 chip->controller->supported_op.cont_read = 1;
5169 static void rawnand_late_check_supported_ops(struct nand_chip *chip)
5171 /* The supported_op fields should not be set by individual drivers */
5172 WARN_ON_ONCE(chip->controller->supported_op.cont_read);
5175 * Too many devices do not support sequential cached reads with on-die
5176 * ECC correction enabled, so in this case refuse to perform the
5179 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE)
5182 if (!nand_has_exec_op(chip))
5185 rawnand_check_cont_read_support(chip);
5189 * Get the flash and manufacturer id and lookup if the type is supported.
5191 static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
5193 const struct nand_manufacturer_desc *manufacturer_desc;
5194 struct mtd_info *mtd = nand_to_mtd(chip);
5195 struct nand_memory_organization *memorg;
5197 u8 *id_data = chip->id.data;
5202 * Let's start by initializing memorg fields that might be left
5203 * unassigned by the ID-based detection logic.
5205 memorg = nanddev_get_memorg(&chip->base);
5206 memorg->planes_per_lun = 1;
5207 memorg->luns_per_target = 1;
5210 * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
5213 ret = nand_reset(chip, 0);
5217 /* Select the device */
5218 nand_select_target(chip, 0);
5220 rawnand_early_check_supported_ops(chip);
5222 /* Send the command for reading device ID */
5223 ret = nand_readid_op(chip, 0, id_data, 2);
5227 /* Read manufacturer and device IDs */
5228 maf_id = id_data[0];
5229 dev_id = id_data[1];
5232 * Try again to make sure, as some systems the bus-hold or other
5233 * interface concerns can cause random data which looks like a
5234 * possibly credible NAND flash to appear. If the two results do
5235 * not match, ignore the device completely.
5238 /* Read entire ID string */
5239 ret = nand_readid_op(chip, 0, id_data, sizeof(chip->id.data));
5243 if (id_data[0] != maf_id || id_data[1] != dev_id) {
5244 pr_info("second ID read did not match %02x,%02x against %02x,%02x\n",
5245 maf_id, dev_id, id_data[0], id_data[1]);
5249 chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data));
5251 /* Try to identify manufacturer */
5252 manufacturer_desc = nand_get_manufacturer_desc(maf_id);
5253 chip->manufacturer.desc = manufacturer_desc;
5256 type = nand_flash_ids;
5259 * Save the NAND_BUSWIDTH_16 flag before letting auto-detection logic
5261 * This is required to make sure initial NAND bus width set by the
5262 * NAND controller driver is coherent with the real NAND bus width
5263 * (extracted by auto-detection code).
5265 busw = chip->options & NAND_BUSWIDTH_16;
5268 * The flag is only set (never cleared), reset it to its default value
5269 * before starting auto-detection.
5271 chip->options &= ~NAND_BUSWIDTH_16;
5273 for (; type->name != NULL; type++) {
5274 if (is_full_id_nand(type)) {
5275 if (find_full_id_nand(chip, type))
5277 } else if (dev_id == type->dev_id) {
5282 if (!type->name || !type->pagesize) {
5283 /* Check if the chip is ONFI compliant */
5284 ret = nand_onfi_detect(chip);
5290 /* Check if the chip is JEDEC compliant */
5291 ret = nand_jedec_detect(chip);
5301 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5302 if (!chip->parameters.model)
5305 if (!type->pagesize)
5306 nand_manufacturer_detect(chip);
5308 nand_decode_id(chip, type);
5310 /* Get chip options */
5311 chip->options |= type->options;
5313 memorg->eraseblocks_per_lun =
5314 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5316 memorg->pages_per_eraseblock);
5320 mtd->name = chip->parameters.model;
5322 if (chip->options & NAND_BUSWIDTH_AUTO) {
5323 WARN_ON(busw & NAND_BUSWIDTH_16);
5324 nand_set_defaults(chip);
5325 } else if (busw != (chip->options & NAND_BUSWIDTH_16)) {
5327 * Check, if buswidth is correct. Hardware drivers should set
5330 pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
5332 pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc),
5334 pr_warn("bus width %d instead of %d bits\n", busw ? 16 : 8,
5335 (chip->options & NAND_BUSWIDTH_16) ? 16 : 8);
5338 goto free_detect_allocation;
5341 nand_decode_bbm_options(chip);
5343 /* Calculate the address shift from the page size */
5344 chip->page_shift = ffs(mtd->writesize) - 1;
5345 /* Convert chipsize to number of pages per chip -1 */
5346 targetsize = nanddev_target_size(&chip->base);
5347 chip->pagemask = (targetsize >> chip->page_shift) - 1;
5349 chip->bbt_erase_shift = chip->phys_erase_shift =
5350 ffs(mtd->erasesize) - 1;
5351 if (targetsize & 0xffffffff)
5352 chip->chip_shift = ffs((unsigned)targetsize) - 1;
5354 chip->chip_shift = ffs((unsigned)(targetsize >> 32));
5355 chip->chip_shift += 32 - 1;
5358 if (chip->chip_shift - chip->page_shift > 16)
5359 chip->options |= NAND_ROW_ADDR_3;
5361 chip->badblockbits = 8;
5363 nand_legacy_adjust_cmdfunc(chip);
5365 pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
5367 pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc),
5368 chip->parameters.model);
5369 pr_info("%d MiB, %s, erase size: %d KiB, page size: %d, OOB size: %d\n",
5370 (int)(targetsize >> 20), nand_is_slc(chip) ? "SLC" : "MLC",
5371 mtd->erasesize >> 10, mtd->writesize, mtd->oobsize);
5374 free_detect_allocation:
5375 kfree(chip->parameters.model);
5380 static enum nand_ecc_engine_type
5381 of_get_rawnand_ecc_engine_type_legacy(struct device_node *np)
5383 enum nand_ecc_legacy_mode {
5389 NAND_ECC_HW_SYNDROME,
5392 const char * const nand_ecc_legacy_modes[] = {
5393 [NAND_ECC_NONE] = "none",
5394 [NAND_ECC_SOFT] = "soft",
5395 [NAND_ECC_SOFT_BCH] = "soft_bch",
5396 [NAND_ECC_HW] = "hw",
5397 [NAND_ECC_HW_SYNDROME] = "hw_syndrome",
5398 [NAND_ECC_ON_DIE] = "on-die",
5400 enum nand_ecc_legacy_mode eng_type;
5404 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5406 return NAND_ECC_ENGINE_TYPE_INVALID;
5408 for (eng_type = NAND_ECC_NONE;
5409 eng_type < ARRAY_SIZE(nand_ecc_legacy_modes); eng_type++) {
5410 if (!strcasecmp(pm, nand_ecc_legacy_modes[eng_type])) {
5413 return NAND_ECC_ENGINE_TYPE_NONE;
5415 case NAND_ECC_SOFT_BCH:
5416 return NAND_ECC_ENGINE_TYPE_SOFT;
5418 case NAND_ECC_HW_SYNDROME:
5419 return NAND_ECC_ENGINE_TYPE_ON_HOST;
5420 case NAND_ECC_ON_DIE:
5421 return NAND_ECC_ENGINE_TYPE_ON_DIE;
5428 return NAND_ECC_ENGINE_TYPE_INVALID;
5431 static enum nand_ecc_placement
5432 of_get_rawnand_ecc_placement_legacy(struct device_node *np)
5437 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5439 if (!strcasecmp(pm, "hw_syndrome"))
5440 return NAND_ECC_PLACEMENT_INTERLEAVED;
5443 return NAND_ECC_PLACEMENT_UNKNOWN;
5446 static enum nand_ecc_algo of_get_rawnand_ecc_algo_legacy(struct device_node *np)
5451 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5453 if (!strcasecmp(pm, "soft"))
5454 return NAND_ECC_ALGO_HAMMING;
5455 else if (!strcasecmp(pm, "soft_bch"))
5456 return NAND_ECC_ALGO_BCH;
5459 return NAND_ECC_ALGO_UNKNOWN;
5462 static void of_get_nand_ecc_legacy_user_config(struct nand_chip *chip)
5464 struct device_node *dn = nand_get_flash_node(chip);
5465 struct nand_ecc_props *user_conf = &chip->base.ecc.user_conf;
5467 if (user_conf->engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5468 user_conf->engine_type = of_get_rawnand_ecc_engine_type_legacy(dn);
5470 if (user_conf->algo == NAND_ECC_ALGO_UNKNOWN)
5471 user_conf->algo = of_get_rawnand_ecc_algo_legacy(dn);
5473 if (user_conf->placement == NAND_ECC_PLACEMENT_UNKNOWN)
5474 user_conf->placement = of_get_rawnand_ecc_placement_legacy(dn);
5477 static int of_get_nand_bus_width(struct nand_chip *chip)
5479 struct device_node *dn = nand_get_flash_node(chip);
5483 ret = of_property_read_u32(dn, "nand-bus-width", &val);
5485 /* Buswidth defaults to 8 if the property does not exist .*/
5491 chip->options |= NAND_BUSWIDTH_16;
5497 static int of_get_nand_secure_regions(struct nand_chip *chip)
5499 struct device_node *dn = nand_get_flash_node(chip);
5500 struct property *prop;
5503 /* Only proceed if the "secure-regions" property is present in DT */
5504 prop = of_find_property(dn, "secure-regions", NULL);
5508 nr_elem = of_property_count_elems_of_size(dn, "secure-regions", sizeof(u64));
5512 chip->nr_secure_regions = nr_elem / 2;
5513 chip->secure_regions = kcalloc(chip->nr_secure_regions, sizeof(*chip->secure_regions),
5515 if (!chip->secure_regions)
5518 for (i = 0, j = 0; i < chip->nr_secure_regions; i++, j += 2) {
5519 of_property_read_u64_index(dn, "secure-regions", j,
5520 &chip->secure_regions[i].offset);
5521 of_property_read_u64_index(dn, "secure-regions", j + 1,
5522 &chip->secure_regions[i].size);
5529 * rawnand_dt_parse_gpio_cs - Parse the gpio-cs property of a controller
5530 * @dev: Device that will be parsed. Also used for managed allocations.
5531 * @cs_array: Array of GPIO desc pointers allocated on success
5532 * @ncs_array: Number of entries in @cs_array updated on success.
5533 * @return 0 on success, an error otherwise.
5535 int rawnand_dt_parse_gpio_cs(struct device *dev, struct gpio_desc ***cs_array,
5536 unsigned int *ncs_array)
5538 struct gpio_desc **descs;
5541 ndescs = gpiod_count(dev, "cs");
5543 dev_dbg(dev, "No valid cs-gpios property\n");
5547 descs = devm_kcalloc(dev, ndescs, sizeof(*descs), GFP_KERNEL);
5551 for (i = 0; i < ndescs; i++) {
5552 descs[i] = gpiod_get_index_optional(dev, "cs", i,
5554 if (IS_ERR(descs[i]))
5555 return PTR_ERR(descs[i]);
5558 *ncs_array = ndescs;
5563 EXPORT_SYMBOL(rawnand_dt_parse_gpio_cs);
5565 static int rawnand_dt_init(struct nand_chip *chip)
5567 struct nand_device *nand = mtd_to_nanddev(nand_to_mtd(chip));
5568 struct device_node *dn = nand_get_flash_node(chip);
5574 ret = of_get_nand_bus_width(chip);
5578 if (of_property_read_bool(dn, "nand-is-boot-medium"))
5579 chip->options |= NAND_IS_BOOT_MEDIUM;
5581 if (of_property_read_bool(dn, "nand-on-flash-bbt"))
5582 chip->bbt_options |= NAND_BBT_USE_FLASH;
5584 of_get_nand_ecc_user_config(nand);
5585 of_get_nand_ecc_legacy_user_config(chip);
5588 * If neither the user nor the NAND controller have requested a specific
5589 * ECC engine type, we will default to NAND_ECC_ENGINE_TYPE_ON_HOST.
5591 nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
5594 * Use the user requested engine type, unless there is none, in this
5595 * case default to the NAND controller choice, otherwise fallback to
5596 * the raw NAND default one.
5598 if (nand->ecc.user_conf.engine_type != NAND_ECC_ENGINE_TYPE_INVALID)
5599 chip->ecc.engine_type = nand->ecc.user_conf.engine_type;
5600 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5601 chip->ecc.engine_type = nand->ecc.defaults.engine_type;
5603 chip->ecc.placement = nand->ecc.user_conf.placement;
5604 chip->ecc.algo = nand->ecc.user_conf.algo;
5605 chip->ecc.strength = nand->ecc.user_conf.strength;
5606 chip->ecc.size = nand->ecc.user_conf.step_size;
5612 * nand_scan_ident - Scan for the NAND device
5613 * @chip: NAND chip object
5614 * @maxchips: number of chips to scan for
5615 * @table: alternative NAND ID table
5617 * This is the first phase of the normal nand_scan() function. It reads the
5618 * flash ID and sets up MTD fields accordingly.
5620 * This helper used to be called directly from controller drivers that needed
5621 * to tweak some ECC-related parameters before nand_scan_tail(). This separation
5622 * prevented dynamic allocations during this phase which was unconvenient and
5623 * as been banned for the benefit of the ->init_ecc()/cleanup_ecc() hooks.
5625 static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips,
5626 struct nand_flash_dev *table)
5628 struct mtd_info *mtd = nand_to_mtd(chip);
5629 struct nand_memory_organization *memorg;
5630 int nand_maf_id, nand_dev_id;
5634 memorg = nanddev_get_memorg(&chip->base);
5636 /* Assume all dies are deselected when we enter nand_scan_ident(). */
5639 mutex_init(&chip->lock);
5640 init_waitqueue_head(&chip->resume_wq);
5642 /* Enforce the right timings for reset/detection */
5643 chip->current_interface_config = nand_get_reset_interface_config();
5645 ret = rawnand_dt_init(chip);
5649 if (!mtd->name && mtd->dev.parent)
5650 mtd->name = dev_name(mtd->dev.parent);
5652 /* Set the default functions */
5653 nand_set_defaults(chip);
5655 ret = nand_legacy_check_hooks(chip);
5659 memorg->ntargets = maxchips;
5661 /* Read the flash type */
5662 ret = nand_detect(chip, table);
5664 if (!(chip->options & NAND_SCAN_SILENT_NODEV))
5665 pr_warn("No NAND device found\n");
5666 nand_deselect_target(chip);
5670 nand_maf_id = chip->id.data[0];
5671 nand_dev_id = chip->id.data[1];
5673 nand_deselect_target(chip);
5675 /* Check for a chip array */
5676 for (i = 1; i < maxchips; i++) {
5679 /* See comment in nand_get_flash_type for reset */
5680 ret = nand_reset(chip, i);
5684 nand_select_target(chip, i);
5685 /* Send the command for reading device ID */
5686 ret = nand_readid_op(chip, 0, id, sizeof(id));
5689 /* Read manufacturer and device IDs */
5690 if (nand_maf_id != id[0] || nand_dev_id != id[1]) {
5691 nand_deselect_target(chip);
5694 nand_deselect_target(chip);
5697 pr_info("%d chips detected\n", i);
5699 /* Store the number of chips and calc total size for mtd */
5700 memorg->ntargets = i;
5701 mtd->size = i * nanddev_target_size(&chip->base);
5706 static void nand_scan_ident_cleanup(struct nand_chip *chip)
5708 kfree(chip->parameters.model);
5709 kfree(chip->parameters.onfi);
5712 int rawnand_sw_hamming_init(struct nand_chip *chip)
5714 struct nand_ecc_sw_hamming_conf *engine_conf;
5715 struct nand_device *base = &chip->base;
5718 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5719 base->ecc.user_conf.algo = NAND_ECC_ALGO_HAMMING;
5720 base->ecc.user_conf.strength = chip->ecc.strength;
5721 base->ecc.user_conf.step_size = chip->ecc.size;
5723 ret = nand_ecc_sw_hamming_init_ctx(base);
5727 engine_conf = base->ecc.ctx.priv;
5729 if (chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER)
5730 engine_conf->sm_order = true;
5732 chip->ecc.size = base->ecc.ctx.conf.step_size;
5733 chip->ecc.strength = base->ecc.ctx.conf.strength;
5734 chip->ecc.total = base->ecc.ctx.total;
5735 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5736 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5740 EXPORT_SYMBOL(rawnand_sw_hamming_init);
5742 int rawnand_sw_hamming_calculate(struct nand_chip *chip,
5743 const unsigned char *buf,
5744 unsigned char *code)
5746 struct nand_device *base = &chip->base;
5748 return nand_ecc_sw_hamming_calculate(base, buf, code);
5750 EXPORT_SYMBOL(rawnand_sw_hamming_calculate);
5752 int rawnand_sw_hamming_correct(struct nand_chip *chip,
5754 unsigned char *read_ecc,
5755 unsigned char *calc_ecc)
5757 struct nand_device *base = &chip->base;
5759 return nand_ecc_sw_hamming_correct(base, buf, read_ecc, calc_ecc);
5761 EXPORT_SYMBOL(rawnand_sw_hamming_correct);
5763 void rawnand_sw_hamming_cleanup(struct nand_chip *chip)
5765 struct nand_device *base = &chip->base;
5767 nand_ecc_sw_hamming_cleanup_ctx(base);
5769 EXPORT_SYMBOL(rawnand_sw_hamming_cleanup);
5771 int rawnand_sw_bch_init(struct nand_chip *chip)
5773 struct nand_device *base = &chip->base;
5774 const struct nand_ecc_props *ecc_conf = nanddev_get_ecc_conf(base);
5777 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5778 base->ecc.user_conf.algo = NAND_ECC_ALGO_BCH;
5779 base->ecc.user_conf.step_size = chip->ecc.size;
5780 base->ecc.user_conf.strength = chip->ecc.strength;
5782 ret = nand_ecc_sw_bch_init_ctx(base);
5786 chip->ecc.size = ecc_conf->step_size;
5787 chip->ecc.strength = ecc_conf->strength;
5788 chip->ecc.total = base->ecc.ctx.total;
5789 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5790 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5794 EXPORT_SYMBOL(rawnand_sw_bch_init);
5796 static int rawnand_sw_bch_calculate(struct nand_chip *chip,
5797 const unsigned char *buf,
5798 unsigned char *code)
5800 struct nand_device *base = &chip->base;
5802 return nand_ecc_sw_bch_calculate(base, buf, code);
5805 int rawnand_sw_bch_correct(struct nand_chip *chip, unsigned char *buf,
5806 unsigned char *read_ecc, unsigned char *calc_ecc)
5808 struct nand_device *base = &chip->base;
5810 return nand_ecc_sw_bch_correct(base, buf, read_ecc, calc_ecc);
5812 EXPORT_SYMBOL(rawnand_sw_bch_correct);
5814 void rawnand_sw_bch_cleanup(struct nand_chip *chip)
5816 struct nand_device *base = &chip->base;
5818 nand_ecc_sw_bch_cleanup_ctx(base);
5820 EXPORT_SYMBOL(rawnand_sw_bch_cleanup);
5822 static int nand_set_ecc_on_host_ops(struct nand_chip *chip)
5824 struct nand_ecc_ctrl *ecc = &chip->ecc;
5826 switch (ecc->placement) {
5827 case NAND_ECC_PLACEMENT_UNKNOWN:
5828 case NAND_ECC_PLACEMENT_OOB:
5829 /* Use standard hwecc read page function? */
5830 if (!ecc->read_page)
5831 ecc->read_page = nand_read_page_hwecc;
5832 if (!ecc->write_page)
5833 ecc->write_page = nand_write_page_hwecc;
5834 if (!ecc->read_page_raw)
5835 ecc->read_page_raw = nand_read_page_raw;
5836 if (!ecc->write_page_raw)
5837 ecc->write_page_raw = nand_write_page_raw;
5839 ecc->read_oob = nand_read_oob_std;
5840 if (!ecc->write_oob)
5841 ecc->write_oob = nand_write_oob_std;
5842 if (!ecc->read_subpage)
5843 ecc->read_subpage = nand_read_subpage;
5844 if (!ecc->write_subpage && ecc->hwctl && ecc->calculate)
5845 ecc->write_subpage = nand_write_subpage_hwecc;
5848 case NAND_ECC_PLACEMENT_INTERLEAVED:
5849 if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) &&
5851 ecc->read_page == nand_read_page_hwecc ||
5853 ecc->write_page == nand_write_page_hwecc)) {
5854 WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
5857 /* Use standard syndrome read/write page function? */
5858 if (!ecc->read_page)
5859 ecc->read_page = nand_read_page_syndrome;
5860 if (!ecc->write_page)
5861 ecc->write_page = nand_write_page_syndrome;
5862 if (!ecc->read_page_raw)
5863 ecc->read_page_raw = nand_read_page_raw_syndrome;
5864 if (!ecc->write_page_raw)
5865 ecc->write_page_raw = nand_write_page_raw_syndrome;
5867 ecc->read_oob = nand_read_oob_syndrome;
5868 if (!ecc->write_oob)
5869 ecc->write_oob = nand_write_oob_syndrome;
5873 pr_warn("Invalid NAND_ECC_PLACEMENT %d\n",
5881 static int nand_set_ecc_soft_ops(struct nand_chip *chip)
5883 struct mtd_info *mtd = nand_to_mtd(chip);
5884 struct nand_device *nanddev = mtd_to_nanddev(mtd);
5885 struct nand_ecc_ctrl *ecc = &chip->ecc;
5888 if (WARN_ON(ecc->engine_type != NAND_ECC_ENGINE_TYPE_SOFT))
5891 switch (ecc->algo) {
5892 case NAND_ECC_ALGO_HAMMING:
5893 ecc->calculate = rawnand_sw_hamming_calculate;
5894 ecc->correct = rawnand_sw_hamming_correct;
5895 ecc->read_page = nand_read_page_swecc;
5896 ecc->read_subpage = nand_read_subpage;
5897 ecc->write_page = nand_write_page_swecc;
5898 if (!ecc->read_page_raw)
5899 ecc->read_page_raw = nand_read_page_raw;
5900 if (!ecc->write_page_raw)
5901 ecc->write_page_raw = nand_write_page_raw;
5902 ecc->read_oob = nand_read_oob_std;
5903 ecc->write_oob = nand_write_oob_std;
5909 if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_HAMMING_SMC))
5910 ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER;
5912 ret = rawnand_sw_hamming_init(chip);
5914 WARN(1, "Hamming ECC initialization failed!\n");
5919 case NAND_ECC_ALGO_BCH:
5920 if (!IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) {
5921 WARN(1, "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n");
5924 ecc->calculate = rawnand_sw_bch_calculate;
5925 ecc->correct = rawnand_sw_bch_correct;
5926 ecc->read_page = nand_read_page_swecc;
5927 ecc->read_subpage = nand_read_subpage;
5928 ecc->write_page = nand_write_page_swecc;
5929 if (!ecc->read_page_raw)
5930 ecc->read_page_raw = nand_read_page_raw;
5931 if (!ecc->write_page_raw)
5932 ecc->write_page_raw = nand_write_page_raw;
5933 ecc->read_oob = nand_read_oob_std;
5934 ecc->write_oob = nand_write_oob_std;
5937 * We can only maximize ECC config when the default layout is
5938 * used, otherwise we don't know how many bytes can really be
5941 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH &&
5942 mtd->ooblayout != nand_get_large_page_ooblayout())
5943 nanddev->ecc.user_conf.flags &= ~NAND_ECC_MAXIMIZE_STRENGTH;
5945 ret = rawnand_sw_bch_init(chip);
5947 WARN(1, "BCH ECC initialization failed!\n");
5953 WARN(1, "Unsupported ECC algorithm!\n");
5959 * nand_check_ecc_caps - check the sanity of preset ECC settings
5960 * @chip: nand chip info structure
5961 * @caps: ECC caps info structure
5962 * @oobavail: OOB size that the ECC engine can use
5964 * When ECC step size and strength are already set, check if they are supported
5965 * by the controller and the calculated ECC bytes fit within the chip's OOB.
5966 * On success, the calculated ECC bytes is set.
5969 nand_check_ecc_caps(struct nand_chip *chip,
5970 const struct nand_ecc_caps *caps, int oobavail)
5972 struct mtd_info *mtd = nand_to_mtd(chip);
5973 const struct nand_ecc_step_info *stepinfo;
5974 int preset_step = chip->ecc.size;
5975 int preset_strength = chip->ecc.strength;
5976 int ecc_bytes, nsteps = mtd->writesize / preset_step;
5979 for (i = 0; i < caps->nstepinfos; i++) {
5980 stepinfo = &caps->stepinfos[i];
5982 if (stepinfo->stepsize != preset_step)
5985 for (j = 0; j < stepinfo->nstrengths; j++) {
5986 if (stepinfo->strengths[j] != preset_strength)
5989 ecc_bytes = caps->calc_ecc_bytes(preset_step,
5991 if (WARN_ON_ONCE(ecc_bytes < 0))
5994 if (ecc_bytes * nsteps > oobavail) {
5995 pr_err("ECC (step, strength) = (%d, %d) does not fit in OOB",
5996 preset_step, preset_strength);
6000 chip->ecc.bytes = ecc_bytes;
6006 pr_err("ECC (step, strength) = (%d, %d) not supported on this controller",
6007 preset_step, preset_strength);
6013 * nand_match_ecc_req - meet the chip's requirement with least ECC bytes
6014 * @chip: nand chip info structure
6015 * @caps: ECC engine caps info structure
6016 * @oobavail: OOB size that the ECC engine can use
6018 * If a chip's ECC requirement is provided, try to meet it with the least
6019 * number of ECC bytes (i.e. with the largest number of OOB-free bytes).
6020 * On success, the chosen ECC settings are set.
6023 nand_match_ecc_req(struct nand_chip *chip,
6024 const struct nand_ecc_caps *caps, int oobavail)
6026 const struct nand_ecc_props *requirements =
6027 nanddev_get_ecc_requirements(&chip->base);
6028 struct mtd_info *mtd = nand_to_mtd(chip);
6029 const struct nand_ecc_step_info *stepinfo;
6030 int req_step = requirements->step_size;
6031 int req_strength = requirements->strength;
6032 int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total;
6033 int best_step = 0, best_strength = 0, best_ecc_bytes = 0;
6034 int best_ecc_bytes_total = INT_MAX;
6037 /* No information provided by the NAND chip */
6038 if (!req_step || !req_strength)
6041 /* number of correctable bits the chip requires in a page */
6042 req_corr = mtd->writesize / req_step * req_strength;
6044 for (i = 0; i < caps->nstepinfos; i++) {
6045 stepinfo = &caps->stepinfos[i];
6046 step_size = stepinfo->stepsize;
6048 for (j = 0; j < stepinfo->nstrengths; j++) {
6049 strength = stepinfo->strengths[j];
6052 * If both step size and strength are smaller than the
6053 * chip's requirement, it is not easy to compare the
6054 * resulted reliability.
6056 if (step_size < req_step && strength < req_strength)
6059 if (mtd->writesize % step_size)
6062 nsteps = mtd->writesize / step_size;
6064 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6065 if (WARN_ON_ONCE(ecc_bytes < 0))
6067 ecc_bytes_total = ecc_bytes * nsteps;
6069 if (ecc_bytes_total > oobavail ||
6070 strength * nsteps < req_corr)
6074 * We assume the best is to meet the chip's requrement
6075 * with the least number of ECC bytes.
6077 if (ecc_bytes_total < best_ecc_bytes_total) {
6078 best_ecc_bytes_total = ecc_bytes_total;
6079 best_step = step_size;
6080 best_strength = strength;
6081 best_ecc_bytes = ecc_bytes;
6086 if (best_ecc_bytes_total == INT_MAX)
6089 chip->ecc.size = best_step;
6090 chip->ecc.strength = best_strength;
6091 chip->ecc.bytes = best_ecc_bytes;
6097 * nand_maximize_ecc - choose the max ECC strength available
6098 * @chip: nand chip info structure
6099 * @caps: ECC engine caps info structure
6100 * @oobavail: OOB size that the ECC engine can use
6102 * Choose the max ECC strength that is supported on the controller, and can fit
6103 * within the chip's OOB. On success, the chosen ECC settings are set.
6106 nand_maximize_ecc(struct nand_chip *chip,
6107 const struct nand_ecc_caps *caps, int oobavail)
6109 struct mtd_info *mtd = nand_to_mtd(chip);
6110 const struct nand_ecc_step_info *stepinfo;
6111 int step_size, strength, nsteps, ecc_bytes, corr;
6114 int best_strength = 0, best_ecc_bytes = 0;
6117 for (i = 0; i < caps->nstepinfos; i++) {
6118 stepinfo = &caps->stepinfos[i];
6119 step_size = stepinfo->stepsize;
6121 /* If chip->ecc.size is already set, respect it */
6122 if (chip->ecc.size && step_size != chip->ecc.size)
6125 for (j = 0; j < stepinfo->nstrengths; j++) {
6126 strength = stepinfo->strengths[j];
6128 if (mtd->writesize % step_size)
6131 nsteps = mtd->writesize / step_size;
6133 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6134 if (WARN_ON_ONCE(ecc_bytes < 0))
6137 if (ecc_bytes * nsteps > oobavail)
6140 corr = strength * nsteps;
6143 * If the number of correctable bits is the same,
6144 * bigger step_size has more reliability.
6146 if (corr > best_corr ||
6147 (corr == best_corr && step_size > best_step)) {
6149 best_step = step_size;
6150 best_strength = strength;
6151 best_ecc_bytes = ecc_bytes;
6159 chip->ecc.size = best_step;
6160 chip->ecc.strength = best_strength;
6161 chip->ecc.bytes = best_ecc_bytes;
6167 * nand_ecc_choose_conf - Set the ECC strength and ECC step size
6168 * @chip: nand chip info structure
6169 * @caps: ECC engine caps info structure
6170 * @oobavail: OOB size that the ECC engine can use
6172 * Choose the ECC configuration according to following logic.
6174 * 1. If both ECC step size and ECC strength are already set (usually by DT)
6175 * then check if it is supported by this controller.
6176 * 2. If the user provided the nand-ecc-maximize property, then select maximum
6178 * 3. Otherwise, try to match the ECC step size and ECC strength closest
6179 * to the chip's requirement. If available OOB size can't fit the chip
6180 * requirement then fallback to the maximum ECC step size and ECC strength.
6182 * On success, the chosen ECC settings are set.
6184 int nand_ecc_choose_conf(struct nand_chip *chip,
6185 const struct nand_ecc_caps *caps, int oobavail)
6187 struct mtd_info *mtd = nand_to_mtd(chip);
6188 struct nand_device *nanddev = mtd_to_nanddev(mtd);
6190 if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize))
6193 if (chip->ecc.size && chip->ecc.strength)
6194 return nand_check_ecc_caps(chip, caps, oobavail);
6196 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH)
6197 return nand_maximize_ecc(chip, caps, oobavail);
6199 if (!nand_match_ecc_req(chip, caps, oobavail))
6202 return nand_maximize_ecc(chip, caps, oobavail);
6204 EXPORT_SYMBOL_GPL(nand_ecc_choose_conf);
6206 static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos)
6208 struct nand_chip *chip = container_of(nand, struct nand_chip,
6210 unsigned int eb = nanddev_pos_to_row(nand, pos);
6213 eb >>= nand->rowconv.eraseblock_addr_shift;
6215 nand_select_target(chip, pos->target);
6216 ret = nand_erase_op(chip, eb);
6217 nand_deselect_target(chip);
6222 static int rawnand_markbad(struct nand_device *nand,
6223 const struct nand_pos *pos)
6225 struct nand_chip *chip = container_of(nand, struct nand_chip,
6228 return nand_markbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6231 static bool rawnand_isbad(struct nand_device *nand, const struct nand_pos *pos)
6233 struct nand_chip *chip = container_of(nand, struct nand_chip,
6237 nand_select_target(chip, pos->target);
6238 ret = nand_isbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6239 nand_deselect_target(chip);
6244 static const struct nand_ops rawnand_ops = {
6245 .erase = rawnand_erase,
6246 .markbad = rawnand_markbad,
6247 .isbad = rawnand_isbad,
6251 * nand_scan_tail - Scan for the NAND device
6252 * @chip: NAND chip object
6254 * This is the second phase of the normal nand_scan() function. It fills out
6255 * all the uninitialized function pointers with the defaults and scans for a
6256 * bad block table if appropriate.
6258 static int nand_scan_tail(struct nand_chip *chip)
6260 struct mtd_info *mtd = nand_to_mtd(chip);
6261 struct nand_ecc_ctrl *ecc = &chip->ecc;
6264 /* New bad blocks should be marked in OOB, flash-based BBT, or both */
6265 if (WARN_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
6266 !(chip->bbt_options & NAND_BBT_USE_FLASH))) {
6270 chip->data_buf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
6271 if (!chip->data_buf)
6275 * FIXME: some NAND manufacturer drivers expect the first die to be
6276 * selected when manufacturer->init() is called. They should be fixed
6277 * to explictly select the relevant die when interacting with the NAND
6280 nand_select_target(chip, 0);
6281 ret = nand_manufacturer_init(chip);
6282 nand_deselect_target(chip);
6286 /* Set the internal oob buffer location, just after the page data */
6287 chip->oob_poi = chip->data_buf + mtd->writesize;
6290 * If no default placement scheme is given, select an appropriate one.
6292 if (!mtd->ooblayout &&
6293 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6294 ecc->algo == NAND_ECC_ALGO_BCH) &&
6295 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6296 ecc->algo == NAND_ECC_ALGO_HAMMING)) {
6297 switch (mtd->oobsize) {
6300 mtd_set_ooblayout(mtd, nand_get_small_page_ooblayout());
6304 mtd_set_ooblayout(mtd,
6305 nand_get_large_page_hamming_ooblayout());
6309 * Expose the whole OOB area to users if ECC_NONE
6310 * is passed. We could do that for all kind of
6311 * ->oobsize, but we must keep the old large/small
6312 * page with ECC layout when ->oobsize <= 128 for
6313 * compatibility reasons.
6315 if (ecc->engine_type == NAND_ECC_ENGINE_TYPE_NONE) {
6316 mtd_set_ooblayout(mtd,
6317 nand_get_large_page_ooblayout());
6321 WARN(1, "No oob scheme defined for oobsize %d\n",
6324 goto err_nand_manuf_cleanup;
6329 * Check ECC mode, default to software if 3byte/512byte hardware ECC is
6330 * selected and we have 256 byte pagesize fallback to software ECC
6333 switch (ecc->engine_type) {
6334 case NAND_ECC_ENGINE_TYPE_ON_HOST:
6335 ret = nand_set_ecc_on_host_ops(chip);
6337 goto err_nand_manuf_cleanup;
6339 if (mtd->writesize >= ecc->size) {
6340 if (!ecc->strength) {
6341 WARN(1, "Driver must set ecc.strength when using hardware ECC\n");
6343 goto err_nand_manuf_cleanup;
6347 pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n",
6348 ecc->size, mtd->writesize);
6349 ecc->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
6350 ecc->algo = NAND_ECC_ALGO_HAMMING;
6353 case NAND_ECC_ENGINE_TYPE_SOFT:
6354 ret = nand_set_ecc_soft_ops(chip);
6356 goto err_nand_manuf_cleanup;
6359 case NAND_ECC_ENGINE_TYPE_ON_DIE:
6360 if (!ecc->read_page || !ecc->write_page) {
6361 WARN(1, "No ECC functions supplied; on-die ECC not possible\n");
6363 goto err_nand_manuf_cleanup;
6366 ecc->read_oob = nand_read_oob_std;
6367 if (!ecc->write_oob)
6368 ecc->write_oob = nand_write_oob_std;
6371 case NAND_ECC_ENGINE_TYPE_NONE:
6372 pr_warn("NAND_ECC_ENGINE_TYPE_NONE selected by board driver. This is not recommended!\n");
6373 ecc->read_page = nand_read_page_raw;
6374 ecc->write_page = nand_write_page_raw;
6375 ecc->read_oob = nand_read_oob_std;
6376 ecc->read_page_raw = nand_read_page_raw;
6377 ecc->write_page_raw = nand_write_page_raw;
6378 ecc->write_oob = nand_write_oob_std;
6379 ecc->size = mtd->writesize;
6385 WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->engine_type);
6387 goto err_nand_manuf_cleanup;
6390 if (ecc->correct || ecc->calculate) {
6391 ecc->calc_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6392 ecc->code_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6393 if (!ecc->calc_buf || !ecc->code_buf) {
6395 goto err_nand_manuf_cleanup;
6399 /* For many systems, the standard OOB write also works for raw */
6400 if (!ecc->read_oob_raw)
6401 ecc->read_oob_raw = ecc->read_oob;
6402 if (!ecc->write_oob_raw)
6403 ecc->write_oob_raw = ecc->write_oob;
6405 /* propagate ecc info to mtd_info */
6406 mtd->ecc_strength = ecc->strength;
6407 mtd->ecc_step_size = ecc->size;
6410 * Set the number of read / write steps for one page depending on ECC
6414 ecc->steps = mtd->writesize / ecc->size;
6415 if (ecc->steps * ecc->size != mtd->writesize) {
6416 WARN(1, "Invalid ECC parameters\n");
6418 goto err_nand_manuf_cleanup;
6422 ecc->total = ecc->steps * ecc->bytes;
6423 chip->base.ecc.ctx.total = ecc->total;
6426 if (ecc->total > mtd->oobsize) {
6427 WARN(1, "Total number of ECC bytes exceeded oobsize\n");
6429 goto err_nand_manuf_cleanup;
6433 * The number of bytes available for a client to place data into
6434 * the out of band area.
6436 ret = mtd_ooblayout_count_freebytes(mtd);
6440 mtd->oobavail = ret;
6442 /* ECC sanity check: warn if it's too weak */
6443 if (!nand_ecc_is_strong_enough(&chip->base))
6444 pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n",
6445 mtd->name, chip->ecc.strength, chip->ecc.size,
6446 nanddev_get_ecc_requirements(&chip->base)->strength,
6447 nanddev_get_ecc_requirements(&chip->base)->step_size);
6449 /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
6450 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) {
6451 switch (ecc->steps) {
6453 mtd->subpage_sft = 1;
6458 mtd->subpage_sft = 2;
6462 chip->subpagesize = mtd->writesize >> mtd->subpage_sft;
6464 /* Invalidate the pagebuffer reference */
6465 chip->pagecache.page = -1;
6467 /* Large page NAND with SOFT_ECC should support subpage reads */
6468 switch (ecc->engine_type) {
6469 case NAND_ECC_ENGINE_TYPE_SOFT:
6470 if (chip->page_shift > 9)
6471 chip->options |= NAND_SUBPAGE_READ;
6478 ret = nanddev_init(&chip->base, &rawnand_ops, mtd->owner);
6480 goto err_nand_manuf_cleanup;
6482 /* Adjust the MTD_CAP_ flags when NAND_ROM is set. */
6483 if (chip->options & NAND_ROM)
6484 mtd->flags = MTD_CAP_ROM;
6486 /* Fill in remaining MTD driver data */
6487 mtd->_erase = nand_erase;
6489 mtd->_unpoint = NULL;
6490 mtd->_panic_write = panic_nand_write;
6491 mtd->_read_oob = nand_read_oob;
6492 mtd->_write_oob = nand_write_oob;
6493 mtd->_sync = nand_sync;
6494 mtd->_lock = nand_lock;
6495 mtd->_unlock = nand_unlock;
6496 mtd->_suspend = nand_suspend;
6497 mtd->_resume = nand_resume;
6498 mtd->_reboot = nand_shutdown;
6499 mtd->_block_isreserved = nand_block_isreserved;
6500 mtd->_block_isbad = nand_block_isbad;
6501 mtd->_block_markbad = nand_block_markbad;
6502 mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks;
6505 * Initialize bitflip_threshold to its default prior scan_bbt() call.
6506 * scan_bbt() might invoke mtd_read(), thus bitflip_threshold must be
6509 if (!mtd->bitflip_threshold)
6510 mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
6512 /* Find the fastest data interface for this chip */
6513 ret = nand_choose_interface_config(chip);
6515 goto err_nanddev_cleanup;
6517 /* Enter fastest possible mode on all dies. */
6518 for (i = 0; i < nanddev_ntargets(&chip->base); i++) {
6519 ret = nand_setup_interface(chip, i);
6521 goto err_free_interface_config;
6524 rawnand_late_check_supported_ops(chip);
6527 * Look for secure regions in the NAND chip. These regions are supposed
6528 * to be protected by a secure element like Trustzone. So the read/write
6529 * accesses to these regions will be blocked in the runtime by this
6532 ret = of_get_nand_secure_regions(chip);
6534 goto err_free_interface_config;
6536 /* Check, if we should skip the bad block table scan */
6537 if (chip->options & NAND_SKIP_BBTSCAN)
6540 /* Build bad block table */
6541 ret = nand_create_bbt(chip);
6543 goto err_free_secure_regions;
6547 err_free_secure_regions:
6548 kfree(chip->secure_regions);
6550 err_free_interface_config:
6551 kfree(chip->best_interface_config);
6553 err_nanddev_cleanup:
6554 nanddev_cleanup(&chip->base);
6556 err_nand_manuf_cleanup:
6557 nand_manufacturer_cleanup(chip);
6560 kfree(chip->data_buf);
6561 kfree(ecc->code_buf);
6562 kfree(ecc->calc_buf);
6567 static int nand_attach(struct nand_chip *chip)
6569 if (chip->controller->ops && chip->controller->ops->attach_chip)
6570 return chip->controller->ops->attach_chip(chip);
6575 static void nand_detach(struct nand_chip *chip)
6577 if (chip->controller->ops && chip->controller->ops->detach_chip)
6578 chip->controller->ops->detach_chip(chip);
6582 * nand_scan_with_ids - [NAND Interface] Scan for the NAND device
6583 * @chip: NAND chip object
6584 * @maxchips: number of chips to scan for.
6585 * @ids: optional flash IDs table
6587 * This fills out all the uninitialized function pointers with the defaults.
6588 * The flash ID is read and the mtd/chip structures are filled with the
6589 * appropriate values.
6591 int nand_scan_with_ids(struct nand_chip *chip, unsigned int maxchips,
6592 struct nand_flash_dev *ids)
6599 ret = nand_scan_ident(chip, maxchips, ids);
6603 ret = nand_attach(chip);
6607 ret = nand_scan_tail(chip);
6616 nand_scan_ident_cleanup(chip);
6620 EXPORT_SYMBOL(nand_scan_with_ids);
6623 * nand_cleanup - [NAND Interface] Free resources held by the NAND device
6624 * @chip: NAND chip object
6626 void nand_cleanup(struct nand_chip *chip)
6628 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT) {
6629 if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING)
6630 rawnand_sw_hamming_cleanup(chip);
6631 else if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
6632 rawnand_sw_bch_cleanup(chip);
6635 nanddev_cleanup(&chip->base);
6637 /* Free secure regions data */
6638 kfree(chip->secure_regions);
6640 /* Free bad block table memory */
6642 kfree(chip->data_buf);
6643 kfree(chip->ecc.code_buf);
6644 kfree(chip->ecc.calc_buf);
6646 /* Free bad block descriptor memory */
6647 if (chip->badblock_pattern && chip->badblock_pattern->options
6648 & NAND_BBT_DYNAMICSTRUCT)
6649 kfree(chip->badblock_pattern);
6651 /* Free the data interface */
6652 kfree(chip->best_interface_config);
6654 /* Free manufacturer priv data. */
6655 nand_manufacturer_cleanup(chip);
6657 /* Free controller specific allocations after chip identification */
6660 /* Free identification phase allocations */
6661 nand_scan_ident_cleanup(chip);
6664 EXPORT_SYMBOL_GPL(nand_cleanup);
6666 MODULE_LICENSE("GPL");
6667 MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>");
6668 MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>");
6669 MODULE_DESCRIPTION("Generic NAND flash driver code");