diff options
Diffstat (limited to 'drivers/mtd/nand/atmel')
-rw-r--r-- | drivers/mtd/nand/atmel/Makefile | 4 | ||||
-rw-r--r-- | drivers/mtd/nand/atmel/nand-controller.c | 2197 | ||||
-rw-r--r-- | drivers/mtd/nand/atmel/pmecc.c | 1020 | ||||
-rw-r--r-- | drivers/mtd/nand/atmel/pmecc.h | 73 |
4 files changed, 3294 insertions, 0 deletions
diff --git a/drivers/mtd/nand/atmel/Makefile b/drivers/mtd/nand/atmel/Makefile new file mode 100644 index 000000000000..288db4f38a8f --- /dev/null +++ b/drivers/mtd/nand/atmel/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_MTD_NAND_ATMEL) += atmel-nand-controller.o atmel-pmecc.o + +atmel-nand-controller-objs := nand-controller.o +atmel-pmecc-objs := pmecc.o diff --git a/drivers/mtd/nand/atmel/nand-controller.c b/drivers/mtd/nand/atmel/nand-controller.c new file mode 100644 index 000000000000..3b2446896147 --- /dev/null +++ b/drivers/mtd/nand/atmel/nand-controller.c @@ -0,0 +1,2197 @@ +/* + * Copyright 2017 ATMEL + * Copyright 2017 Free Electrons + * + * Author: Boris Brezillon <boris.brezillon@free-electrons.com> + * + * Derived from the atmel_nand.c driver which contained the following + * copyrights: + * + * Copyright 2003 Rick Bronson + * + * Derived from drivers/mtd/nand/autcpu12.c + * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) + * + * Derived from drivers/mtd/spia.c + * Copyright 2000 Steven J. Hill (sjhill@cotw.com) + * + * + * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 + * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 + * + * Derived from Das U-Boot source code + * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) + * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas + * + * Add Programmable Multibit ECC support for various AT91 SoC + * Copyright 2012 ATMEL, Hong Xu + * + * Add Nand Flash Controller support for SAMA5 SoC + * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * A few words about the naming convention in this file. This convention + * applies to structure and function names. + * + * Prefixes: + * + * - atmel_nand_: all generic structures/functions + * - atmel_smc_nand_: all structures/functions specific to the SMC interface + * (at91sam9 and avr32 SoCs) + * - atmel_hsmc_nand_: all structures/functions specific to the HSMC interface + * (sama5 SoCs and later) + * - atmel_nfc_: all structures/functions used to manipulate the NFC sub-block + * that is available in the HSMC block + * - <soc>_nand_: all SoC specific structures/functions + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/genalloc.h> +#include <linux/gpio.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/mfd/syscon.h> +#include <linux/mfd/syscon/atmel-matrix.h> +#include <linux/module.h> +#include <linux/mtd/nand.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/iopoll.h> +#include <linux/platform_device.h> +#include <linux/platform_data/atmel.h> +#include <linux/regmap.h> + +#include "pmecc.h" + +#define ATMEL_HSMC_NFC_CFG 0x0 +#define ATMEL_HSMC_NFC_CFG_SPARESIZE(x) (((x) / 4) << 24) +#define ATMEL_HSMC_NFC_CFG_SPARESIZE_MASK GENMASK(30, 24) +#define ATMEL_HSMC_NFC_CFG_DTO(cyc, mul) (((cyc) << 16) | ((mul) << 20)) +#define ATMEL_HSMC_NFC_CFG_DTO_MAX GENMASK(22, 16) +#define ATMEL_HSMC_NFC_CFG_RBEDGE BIT(13) +#define ATMEL_HSMC_NFC_CFG_FALLING_EDGE BIT(12) +#define ATMEL_HSMC_NFC_CFG_RSPARE BIT(9) +#define ATMEL_HSMC_NFC_CFG_WSPARE BIT(8) +#define ATMEL_HSMC_NFC_CFG_PAGESIZE_MASK GENMASK(2, 0) +#define ATMEL_HSMC_NFC_CFG_PAGESIZE(x) (fls((x) / 512) - 1) + +#define ATMEL_HSMC_NFC_CTRL 0x4 +#define ATMEL_HSMC_NFC_CTRL_EN BIT(0) +#define ATMEL_HSMC_NFC_CTRL_DIS BIT(1) + +#define ATMEL_HSMC_NFC_SR 0x8 +#define ATMEL_HSMC_NFC_IER 0xc +#define ATMEL_HSMC_NFC_IDR 0x10 +#define ATMEL_HSMC_NFC_IMR 0x14 +#define ATMEL_HSMC_NFC_SR_ENABLED BIT(1) +#define ATMEL_HSMC_NFC_SR_RB_RISE BIT(4) +#define ATMEL_HSMC_NFC_SR_RB_FALL BIT(5) +#define ATMEL_HSMC_NFC_SR_BUSY BIT(8) +#define ATMEL_HSMC_NFC_SR_WR BIT(11) +#define ATMEL_HSMC_NFC_SR_CSID GENMASK(14, 12) +#define ATMEL_HSMC_NFC_SR_XFRDONE BIT(16) +#define ATMEL_HSMC_NFC_SR_CMDDONE BIT(17) +#define ATMEL_HSMC_NFC_SR_DTOE BIT(20) +#define ATMEL_HSMC_NFC_SR_UNDEF BIT(21) +#define ATMEL_HSMC_NFC_SR_AWB BIT(22) +#define ATMEL_HSMC_NFC_SR_NFCASE BIT(23) +#define ATMEL_HSMC_NFC_SR_ERRORS (ATMEL_HSMC_NFC_SR_DTOE | \ + ATMEL_HSMC_NFC_SR_UNDEF | \ + ATMEL_HSMC_NFC_SR_AWB | \ + ATMEL_HSMC_NFC_SR_NFCASE) +#define ATMEL_HSMC_NFC_SR_RBEDGE(x) BIT((x) + 24) + +#define ATMEL_HSMC_NFC_ADDR 0x18 +#define ATMEL_HSMC_NFC_BANK 0x1c + +#define ATMEL_NFC_MAX_RB_ID 7 + +#define ATMEL_NFC_SRAM_SIZE 0x2400 + +#define ATMEL_NFC_CMD(pos, cmd) ((cmd) << (((pos) * 8) + 2)) +#define ATMEL_NFC_VCMD2 BIT(18) +#define ATMEL_NFC_ACYCLE(naddrs) ((naddrs) << 19) +#define ATMEL_NFC_CSID(cs) ((cs) << 22) +#define ATMEL_NFC_DATAEN BIT(25) +#define ATMEL_NFC_NFCWR BIT(26) + +#define ATMEL_NFC_MAX_ADDR_CYCLES 5 + +#define ATMEL_NAND_ALE_OFFSET BIT(21) +#define ATMEL_NAND_CLE_OFFSET BIT(22) + +#define DEFAULT_TIMEOUT_MS 1000 +#define MIN_DMA_LEN 128 + +enum atmel_nand_rb_type { + ATMEL_NAND_NO_RB, + ATMEL_NAND_NATIVE_RB, + ATMEL_NAND_GPIO_RB, +}; + +struct atmel_nand_rb { + enum atmel_nand_rb_type type; + union { + struct gpio_desc *gpio; + int id; + }; +}; + +struct atmel_nand_cs { + int id; + struct atmel_nand_rb rb; + struct gpio_desc *csgpio; + struct { + void __iomem *virt; + dma_addr_t dma; + } io; +}; + +struct atmel_nand { + struct list_head node; + struct device *dev; + struct nand_chip base; + struct atmel_nand_cs *activecs; + struct atmel_pmecc_user *pmecc; + struct gpio_desc *cdgpio; + int numcs; + struct atmel_nand_cs cs[]; +}; + +static inline struct atmel_nand *to_atmel_nand(struct nand_chip *chip) +{ + return container_of(chip, struct atmel_nand, base); +} + +enum atmel_nfc_data_xfer { + ATMEL_NFC_NO_DATA, + ATMEL_NFC_READ_DATA, + ATMEL_NFC_WRITE_DATA, +}; + +struct atmel_nfc_op { + u8 cs; + u8 ncmds; + u8 cmds[2]; + u8 naddrs; + u8 addrs[5]; + enum atmel_nfc_data_xfer data; + u32 wait; + u32 errors; +}; + +struct atmel_nand_controller; +struct atmel_nand_controller_caps; + +struct atmel_nand_controller_ops { + int (*probe)(struct platform_device *pdev, + const struct atmel_nand_controller_caps *caps); + int (*remove)(struct atmel_nand_controller *nc); + void (*nand_init)(struct atmel_nand_controller *nc, + struct atmel_nand *nand); + int (*ecc_init)(struct atmel_nand *nand); +}; + +struct atmel_nand_controller_caps { + bool has_dma; + bool legacy_of_bindings; + u32 ale_offs; + u32 cle_offs; + const struct atmel_nand_controller_ops *ops; +}; + +struct atmel_nand_controller { + struct nand_hw_control base; + const struct atmel_nand_controller_caps *caps; + struct device *dev; + struct regmap *smc; + struct dma_chan *dmac; + struct atmel_pmecc *pmecc; + struct list_head chips; + struct clk *mck; +}; + +static inline struct atmel_nand_controller * +to_nand_controller(struct nand_hw_control *ctl) +{ + return container_of(ctl, struct atmel_nand_controller, base); +} + +struct atmel_smc_nand_controller { + struct atmel_nand_controller base; + struct regmap *matrix; + unsigned int ebi_csa_offs; +}; + +static inline struct atmel_smc_nand_controller * +to_smc_nand_controller(struct nand_hw_control *ctl) +{ + return container_of(to_nand_controller(ctl), + struct atmel_smc_nand_controller, base); +} + +struct atmel_hsmc_nand_controller { + struct atmel_nand_controller base; + struct { + struct gen_pool *pool; + void __iomem *virt; + dma_addr_t dma; + } sram; + struct regmap *io; + struct atmel_nfc_op op; + struct completion complete; + int irq; + + /* Only used when instantiating from legacy DT bindings. */ + struct clk *clk; +}; + +static inline struct atmel_hsmc_nand_controller * +to_hsmc_nand_controller(struct nand_hw_control *ctl) +{ + return container_of(to_nand_controller(ctl), + struct atmel_hsmc_nand_controller, base); +} + +static bool atmel_nfc_op_done(struct atmel_nfc_op *op, u32 status) +{ + op->errors |= status & ATMEL_HSMC_NFC_SR_ERRORS; + op->wait ^= status & op->wait; + + return !op->wait || op->errors; +} + +static irqreturn_t atmel_nfc_interrupt(int irq, void *data) +{ + struct atmel_hsmc_nand_controller *nc = data; + u32 sr, rcvd; + bool done; + + regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &sr); + + rcvd = sr & (nc->op.wait | ATMEL_HSMC_NFC_SR_ERRORS); + done = atmel_nfc_op_done(&nc->op, sr); + + if (rcvd) + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_IDR, rcvd); + + if (done) + complete(&nc->complete); + + return rcvd ? IRQ_HANDLED : IRQ_NONE; +} + +static int atmel_nfc_wait(struct atmel_hsmc_nand_controller *nc, bool poll, + unsigned int timeout_ms) +{ + int ret; + + if (!timeout_ms) + timeout_ms = DEFAULT_TIMEOUT_MS; + + if (poll) { + u32 status; + + ret = regmap_read_poll_timeout(nc->base.smc, + ATMEL_HSMC_NFC_SR, status, + atmel_nfc_op_done(&nc->op, + status), + 0, timeout_ms * 1000); + } else { + init_completion(&nc->complete); + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_IER, + nc->op.wait | ATMEL_HSMC_NFC_SR_ERRORS); + ret = wait_for_completion_timeout(&nc->complete, + msecs_to_jiffies(timeout_ms)); + if (!ret) + ret = -ETIMEDOUT; + else + ret = 0; + + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_IDR, 0xffffffff); + } + + if (nc->op.errors & ATMEL_HSMC_NFC_SR_DTOE) { + dev_err(nc->base.dev, "Waiting NAND R/B Timeout\n"); + ret = -ETIMEDOUT; + } + + if (nc->op.errors & ATMEL_HSMC_NFC_SR_UNDEF) { + dev_err(nc->base.dev, "Access to an undefined area\n"); + ret = -EIO; + } + + if (nc->op.errors & ATMEL_HSMC_NFC_SR_AWB) { + dev_err(nc->base.dev, "Access while busy\n"); + ret = -EIO; + } + + if (nc->op.errors & ATMEL_HSMC_NFC_SR_NFCASE) { + dev_err(nc->base.dev, "Wrong access size\n"); + ret = -EIO; + } + + return ret; +} + +static void atmel_nand_dma_transfer_finished(void *data) +{ + struct completion *finished = data; + + complete(finished); +} + +static int atmel_nand_dma_transfer(struct atmel_nand_controller *nc, + void *buf, dma_addr_t dev_dma, size_t len, + enum dma_data_direction dir) +{ + DECLARE_COMPLETION_ONSTACK(finished); + dma_addr_t src_dma, dst_dma, buf_dma; + struct dma_async_tx_descriptor *tx; + dma_cookie_t cookie; + + buf_dma = dma_map_single(nc->dev, buf, len, dir); + if (dma_mapping_error(nc->dev, dev_dma)) { + dev_err(nc->dev, + "Failed to prepare a buffer for DMA access\n"); + goto err; + } + + if (dir == DMA_FROM_DEVICE) { + src_dma = dev_dma; + dst_dma = buf_dma; + } else { + src_dma = buf_dma; + dst_dma = dev_dma; + } + + tx = dmaengine_prep_dma_memcpy(nc->dmac, dst_dma, src_dma, len, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + if (!tx) { + dev_err(nc->dev, "Failed to prepare DMA memcpy\n"); + goto err_unmap; + } + + tx->callback = atmel_nand_dma_transfer_finished; + tx->callback_param = &finished; + + cookie = dmaengine_submit(tx); + if (dma_submit_error(cookie)) { + dev_err(nc->dev, "Failed to do DMA tx_submit\n"); + goto err_unmap; + } + + dma_async_issue_pending(nc->dmac); + wait_for_completion(&finished); + + return 0; + +err_unmap: + dma_unmap_single(nc->dev, buf_dma, len, dir); + +err: + dev_dbg(nc->dev, "Fall back to CPU I/O\n"); + + return -EIO; +} + +static u8 atmel_nand_read_byte(struct mtd_info *mtd) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + + return ioread8(nand->activecs->io.virt); +} + +static u16 atmel_nand_read_word(struct mtd_info *mtd) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + + return ioread16(nand->activecs->io.virt); +} + +static void atmel_nand_write_byte(struct mtd_info *mtd, u8 byte) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + + if (chip->options & NAND_BUSWIDTH_16) + iowrite16(byte | (byte << 8), nand->activecs->io.virt); + else + iowrite8(byte, nand->activecs->io.virt); +} + +static void atmel_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + + nc = to_nand_controller(chip->controller); + + /* + * If the controller supports DMA, the buffer address is DMA-able and + * len is long enough to make DMA transfers profitable, let's trigger + * a DMA transfer. If it fails, fallback to PIO mode. + */ + if (nc->dmac && virt_addr_valid(buf) && + len >= MIN_DMA_LEN && + !atmel_nand_dma_transfer(nc, buf, nand->activecs->io.dma, len, + DMA_FROM_DEVICE)) + return; + + if (chip->options & NAND_BUSWIDTH_16) + ioread16_rep(nand->activecs->io.virt, buf, len / 2); + else + ioread8_rep(nand->activecs->io.virt, buf, len); +} + +static void atmel_nand_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + + nc = to_nand_controller(chip->controller); + + /* + * If the controller supports DMA, the buffer address is DMA-able and + * len is long enough to make DMA transfers profitable, let's trigger + * a DMA transfer. If it fails, fallback to PIO mode. + */ + if (nc->dmac && virt_addr_valid(buf) && + len >= MIN_DMA_LEN && + !atmel_nand_dma_transfer(nc, (void *)buf, nand->activecs->io.dma, + len, DMA_TO_DEVICE)) + return; + + if (chip->options & NAND_BUSWIDTH_16) + iowrite16_rep(nand->activecs->io.virt, buf, len / 2); + else + iowrite8_rep(nand->activecs->io.virt, buf, len); +} + +static int atmel_nand_dev_ready(struct mtd_info *mtd) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + + return gpiod_get_value(nand->activecs->rb.gpio); +} + +static void atmel_nand_select_chip(struct mtd_info *mtd, int cs) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + + if (cs < 0 || cs >= nand->numcs) { + nand->activecs = NULL; + chip->dev_ready = NULL; + return; + } + + nand->activecs = &nand->cs[cs]; + + if (nand->activecs->rb.type == ATMEL_NAND_GPIO_RB) + chip->dev_ready = atmel_nand_dev_ready; +} + +static int atmel_hsmc_nand_dev_ready(struct mtd_info *mtd) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_hsmc_nand_controller *nc; + u32 status; + + nc = to_hsmc_nand_controller(chip->controller); + + regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &status); + + return status & ATMEL_HSMC_NFC_SR_RBEDGE(nand->activecs->rb.id); +} + +static void atmel_hsmc_nand_select_chip(struct mtd_info *mtd, int cs) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_hsmc_nand_controller *nc; + + nc = to_hsmc_nand_controller(chip->controller); + + atmel_nand_select_chip(mtd, cs); + + if (!nand->activecs) { + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, + ATMEL_HSMC_NFC_CTRL_DIS); + return; + } + + if (nand->activecs->rb.type == ATMEL_NAND_NATIVE_RB) + chip->dev_ready = atmel_hsmc_nand_dev_ready; + + regmap_update_bits(nc->base.smc, ATMEL_HSMC_NFC_CFG, + ATMEL_HSMC_NFC_CFG_PAGESIZE_MASK | + ATMEL_HSMC_NFC_CFG_SPARESIZE_MASK | + ATMEL_HSMC_NFC_CFG_RSPARE | + ATMEL_HSMC_NFC_CFG_WSPARE, + ATMEL_HSMC_NFC_CFG_PAGESIZE(mtd->writesize) | + ATMEL_HSMC_NFC_CFG_SPARESIZE(mtd->oobsize) | + ATMEL_HSMC_NFC_CFG_RSPARE); + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, + ATMEL_HSMC_NFC_CTRL_EN); +} + +static int atmel_nfc_exec_op(struct atmel_hsmc_nand_controller *nc, bool poll) +{ + u8 *addrs = nc->op.addrs; + unsigned int op = 0; + u32 addr, val; + int i, ret; + + nc->op.wait = ATMEL_HSMC_NFC_SR_CMDDONE; + + for (i = 0; i < nc->op.ncmds; i++) + op |= ATMEL_NFC_CMD(i, nc->op.cmds[i]); + + if (nc->op.naddrs == ATMEL_NFC_MAX_ADDR_CYCLES) + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_ADDR, *addrs++); + + op |= ATMEL_NFC_CSID(nc->op.cs) | + ATMEL_NFC_ACYCLE(nc->op.naddrs); + + if (nc->op.ncmds > 1) + op |= ATMEL_NFC_VCMD2; + + addr = addrs[0] | (addrs[1] << 8) | (addrs[2] << 16) | + (addrs[3] << 24); + + if (nc->op.data != ATMEL_NFC_NO_DATA) { + op |= ATMEL_NFC_DATAEN; + nc->op.wait |= ATMEL_HSMC_NFC_SR_XFRDONE; + + if (nc->op.data == ATMEL_NFC_WRITE_DATA) + op |= ATMEL_NFC_NFCWR; + } + + /* Clear all flags. */ + regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &val); + + /* Send the command. */ + regmap_write(nc->io, op, addr); + + ret = atmel_nfc_wait(nc, poll, 0); + if (ret) + dev_err(nc->base.dev, + "Failed to send NAND command (err = %d)!", + ret); + + /* Reset the op state. */ + memset(&nc->op, 0, sizeof(nc->op)); + + return ret; +} + +static void atmel_hsmc_nand_cmd_ctrl(struct mtd_info *mtd, int dat, + unsigned int ctrl) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_hsmc_nand_controller *nc; + + nc = to_hsmc_nand_controller(chip->controller); + + if (ctrl & NAND_ALE) { + if (nc->op.naddrs == ATMEL_NFC_MAX_ADDR_CYCLES) + return; + + nc->op.addrs[nc->op.naddrs++] = dat; + } else if (ctrl & NAND_CLE) { + if (nc->op.ncmds > 1) + return; + + nc->op.cmds[nc->op.ncmds++] = dat; + } + + if (dat == NAND_CMD_NONE) { + nc->op.cs = nand->activecs->id; + atmel_nfc_exec_op(nc, true); + } +} + +static void atmel_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, + unsigned int ctrl) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + + nc = to_nand_controller(chip->controller); + + if ((ctrl & NAND_CTRL_CHANGE) && nand->activecs->csgpio) { + if (ctrl & NAND_NCE) + gpiod_set_value(nand->activecs->csgpio, 0); + else + gpiod_set_value(nand->activecs->csgpio, 1); + } + + if (ctrl & NAND_ALE) + writeb(cmd, nand->activecs->io.virt + nc->caps->ale_offs); + else if (ctrl & NAND_CLE) + writeb(cmd, nand->activecs->io.virt + nc->caps->cle_offs); +} + +static void atmel_nfc_copy_to_sram(struct nand_chip *chip, const u8 *buf, + bool oob_required) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_hsmc_nand_controller *nc; + int ret = -EIO; + + nc = to_hsmc_nand_controller(chip->controller); + + if (nc->base.dmac) + ret = atmel_nand_dma_transfer(&nc->base, (void *)buf, + nc->sram.dma, mtd->writesize, + DMA_TO_DEVICE); + + /* Falling back to CPU copy. */ + if (ret) + memcpy_toio(nc->sram.virt, buf, mtd->writesize); + + if (oob_required) + memcpy_toio(nc->sram.virt + mtd->writesize, chip->oob_poi, + mtd->oobsize); +} + +static void atmel_nfc_copy_from_sram(struct nand_chip *chip, u8 *buf, + bool oob_required) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_hsmc_nand_controller *nc; + int ret = -EIO; + + nc = to_hsmc_nand_controller(chip->controller); + + if (nc->base.dmac) + ret = atmel_nand_dma_transfer(&nc->base, buf, nc->sram.dma, + mtd->writesize, DMA_FROM_DEVICE); + + /* Falling back to CPU copy. */ + if (ret) + memcpy_fromio(buf, nc->sram.virt, mtd->writesize); + + if (oob_required) + memcpy_fromio(chip->oob_poi, nc->sram.virt + mtd->writesize, + mtd->oobsize); +} + +static void atmel_nfc_set_op_addr(struct nand_chip *chip, int page, int column) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_hsmc_nand_controller *nc; + + nc = to_hsmc_nand_controller(chip->controller); + + if (column >= 0) { + nc->op.addrs[nc->op.naddrs++] = column; + + /* + * 2 address cycles for the column offset on large page NANDs. + */ + if (mtd->writesize > 512) + nc->op.addrs[nc->op.naddrs++] = column >> 8; + } + + if (page >= 0) { + nc->op.addrs[nc->op.naddrs++] = page; + nc->op.addrs[nc->op.naddrs++] = page >> 8; + + if ((mtd->writesize > 512 && chip->chipsize > SZ_128M) || + (mtd->writesize <= 512 && chip->chipsize > SZ_32M)) + nc->op.addrs[nc->op.naddrs++] = page >> 16; + } +} + +static int atmel_nand_pmecc_enable(struct nand_chip *chip, int op, bool raw) +{ + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + int ret; + + nc = to_nand_controller(chip->controller); + + if (raw) + return 0; + + ret = atmel_pmecc_enable(nand->pmecc, op); + if (ret) + dev_err(nc->dev, + "Failed to enable ECC engine (err = %d)\n", ret); + + return ret; +} + +static void atmel_nand_pmecc_disable(struct nand_chip *chip, bool raw) +{ + struct atmel_nand *nand = to_atmel_nand(chip); + + if (!raw) + atmel_pmecc_disable(nand->pmecc); +} + +static int atmel_nand_pmecc_generate_eccbytes(struct nand_chip *chip, bool raw) +{ + struct atmel_nand *nand = to_atmel_nand(chip); + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand_controller *nc; + struct mtd_oob_region oobregion; + void *eccbuf; + int ret, i; + + nc = to_nand_controller(chip->controller); + + if (raw) + return 0; + + ret = atmel_pmecc_wait_rdy(nand->pmecc); + if (ret) { + dev_err(nc->dev, + "Failed to transfer NAND page data (err = %d)\n", + ret); + return ret; + } + + mtd_ooblayout_ecc(mtd, 0, &oobregion); + eccbuf = chip->oob_poi + oobregion.offset; + + for (i = 0; i < chip->ecc.steps; i++) { + atmel_pmecc_get_generated_eccbytes(nand->pmecc, i, + eccbuf); + eccbuf += chip->ecc.bytes; + } + + return 0; +} + +static int atmel_nand_pmecc_correct_data(struct nand_chip *chip, void *buf, + bool raw) +{ + struct atmel_nand *nand = to_atmel_nand(chip); + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand_controller *nc; + struct mtd_oob_region oobregion; + int ret, i, max_bitflips = 0; + void *databuf, *eccbuf; + + nc = to_nand_controller(chip->controller); + + if (raw) + return 0; + + ret = atmel_pmecc_wait_rdy(nand->pmecc); + if (ret) { + dev_err(nc->dev, + "Failed to read NAND page data (err = %d)\n", + ret); + return ret; + } + + mtd_ooblayout_ecc(mtd, 0, &oobregion); + eccbuf = chip->oob_poi + oobregion.offset; + databuf = buf; + + for (i = 0; i < chip->ecc.steps; i++) { + ret = atmel_pmecc_correct_sector(nand->pmecc, i, databuf, + eccbuf); + if (ret < 0 && !atmel_pmecc_correct_erased_chunks(nand->pmecc)) + ret = nand_check_erased_ecc_chunk(databuf, + chip->ecc.size, + eccbuf, + chip->ecc.bytes, + NULL, 0, + chip->ecc.strength); + + if (ret >= 0) + max_bitflips = max(ret, max_bitflips); + else + mtd->ecc_stats.failed++; + + databuf += chip->ecc.size; + eccbuf += chip->ecc.bytes; + } + + return max_bitflips; +} + +static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf, + bool oob_required, int page, bool raw) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand *nand = to_atmel_nand(chip); + int ret; + + ret = atmel_nand_pmecc_enable(chip, NAND_ECC_WRITE, raw); + if (ret) + return ret; + + atmel_nand_write_buf(mtd, buf, mtd->writesize); + + ret = atmel_nand_pmecc_generate_eccbytes(chip, raw); + if (ret) { + atmel_pmecc_disable(nand->pmecc); + return ret; + } + + atmel_nand_pmecc_disable(chip, raw); + + atmel_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize); + + return 0; +} + +static int atmel_nand_pmecc_write_page(struct mtd_info *mtd, + struct nand_chip *chip, const u8 *buf, + int oob_required, int page) +{ + return atmel_nand_pmecc_write_pg(chip, buf, oob_required, page, false); +} + +static int atmel_nand_pmecc_write_page_raw(struct mtd_info *mtd, + struct nand_chip *chip, + const u8 *buf, int oob_required, + int page) +{ + return atmel_nand_pmecc_write_pg(chip, buf, oob_required, page, true); +} + +static int atmel_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, + bool oob_required, int page, bool raw) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ret; + + ret = atmel_nand_pmecc_enable(chip, NAND_ECC_READ, raw); + if (ret) + return ret; + + atmel_nand_read_buf(mtd, buf, mtd->writesize); + atmel_nand_read_buf(mtd, chip->oob_poi, mtd->oobsize); + + ret = atmel_nand_pmecc_correct_data(chip, buf, raw); + + atmel_nand_pmecc_disable(chip, raw); + + return ret; +} + +static int atmel_nand_pmecc_read_page(struct mtd_info *mtd, + struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, false); +} + +static int atmel_nand_pmecc_read_page_raw(struct mtd_info *mtd, + struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, true); +} + +static int atmel_hsmc_nand_pmecc_write_pg(struct nand_chip *chip, + const u8 *buf, bool oob_required, + int page, bool raw) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_hsmc_nand_controller *nc; + int ret; + + nc = to_hsmc_nand_controller(chip->controller); + + atmel_nfc_copy_to_sram(chip, buf, false); + + nc->op.cmds[0] = NAND_CMD_SEQIN; + nc->op.ncmds = 1; + atmel_nfc_set_op_addr(chip, page, 0x0); + nc->op.cs = nand->activecs->id; + nc->op.data = ATMEL_NFC_WRITE_DATA; + + ret = atmel_nand_pmecc_enable(chip, NAND_ECC_WRITE, raw); + if (ret) + return ret; + + ret = atmel_nfc_exec_op(nc, false); + if (ret) { + atmel_nand_pmecc_disable(chip, raw); + dev_err(nc->base.dev, + "Failed to transfer NAND page data (err = %d)\n", + ret); + return ret; + } + + ret = atmel_nand_pmecc_generate_eccbytes(chip, raw); + + atmel_nand_pmecc_disable(chip, raw); + + if (ret) + return ret; + + atmel_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize); + + nc->op.cmds[0] = NAND_CMD_PAGEPROG; + nc->op.ncmds = 1; + nc->op.cs = nand->activecs->id; + ret = atmel_nfc_exec_op(nc, false); + if (ret) + dev_err(nc->base.dev, "Failed to program NAND page (err = %d)\n", + ret); + + return ret; +} + +static int atmel_hsmc_nand_pmecc_write_page(struct mtd_info *mtd, + struct nand_chip *chip, + const u8 *buf, int oob_required, + int page) +{ + return atmel_hsmc_nand_pmecc_write_pg(chip, buf, oob_required, page, + false); +} + +static int atmel_hsmc_nand_pmecc_write_page_raw(struct mtd_info *mtd, + struct nand_chip *chip, + const u8 *buf, + int oob_required, int page) +{ + return atmel_hsmc_nand_pmecc_write_pg(chip, buf, oob_required, page, + true); +} + +static int atmel_hsmc_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, + bool oob_required, int page, + bool raw) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_hsmc_nand_controller *nc; + int ret; + + nc = to_hsmc_nand_controller(chip->controller); + + /* + * Optimized read page accessors only work when the NAND R/B pin is + * connected to a native SoC R/B pin. If that's not the case, fallback + * to the non-optimized one. + */ + if (nand->activecs->rb.type != ATMEL_NAND_NATIVE_RB) { + chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page); + + return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, + raw); + } + + nc->op.cmds[nc->op.ncmds++] = NAND_CMD_READ0; + + if (mtd->writesize > 512) + nc->op.cmds[nc->op.ncmds++] = NAND_CMD_READSTART; + + atmel_nfc_set_op_addr(chip, page, 0x0); + nc->op.cs = nand->activecs->id; + nc->op.data = ATMEL_NFC_READ_DATA; + + ret = atmel_nand_pmecc_enable(chip, NAND_ECC_READ, raw); + if (ret) + return ret; + + ret = atmel_nfc_exec_op(nc, false); + if (ret) { + atmel_nand_pmecc_disable(chip, raw); + dev_err(nc->base.dev, + "Failed to load NAND page data (err = %d)\n", + ret); + return ret; + } + + atmel_nfc_copy_from_sram(chip, buf, true); + + ret = atmel_nand_pmecc_correct_data(chip, buf, raw); + + atmel_nand_pmecc_disable(chip, raw); + + return ret; +} + +static int atmel_hsmc_nand_pmecc_read_page(struct mtd_info *mtd, + struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + return atmel_hsmc_nand_pmecc_read_pg(chip, buf, oob_required, page, + false); +} + +static int atmel_hsmc_nand_pmecc_read_page_raw(struct mtd_info *mtd, + struct nand_chip *chip, + u8 *buf, int oob_required, + int page) +{ + return atmel_hsmc_nand_pmecc_read_pg(chip, buf, oob_required, page, + true); +} + +static int atmel_nand_pmecc_init(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + struct atmel_pmecc_user_req req; + + nc = to_nand_controller(chip->controller); + + if (!nc->pmecc) { + dev_err(nc->dev, "HW ECC not supported\n"); + return -ENOTSUPP; + } + + if (nc->caps->legacy_of_bindings) { + u32 val; + + if (!of_property_read_u32(nc->dev->of_node, "atmel,pmecc-cap", + &val)) + chip->ecc.strength = val; + + if (!of_property_read_u32(nc->dev->of_node, + "atmel,pmecc-sector-size", + &val)) + chip->ecc.size = val; + } + + if (chip->ecc.options & NAND_ECC_MAXIMIZE) + req.ecc.strength = ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH; + else if (chip->ecc.strength) + req.ecc.strength = chip->ecc.strength; + else if (chip->ecc_strength_ds) + req.ecc.strength = chip->ecc_strength_ds; + else + req.ecc.strength = ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH; + + if (chip->ecc.size) + req.ecc.sectorsize = chip->ecc.size; + else if (chip->ecc_step_ds) + req.ecc.sectorsize = chip->ecc_step_ds; + else + req.ecc.sectorsize = ATMEL_PMECC_SECTOR_SIZE_AUTO; + + req.pagesize = mtd->writesize; + req.oobsize = mtd->oobsize; + + if (mtd->writesize <= 512) { + req.ecc.bytes = 4; + req.ecc.ooboffset = 0; + } else { + req.ecc.bytes = mtd->oobsize - 2; + req.ecc.ooboffset = ATMEL_PMECC_OOBOFFSET_AUTO; + } + + nand->pmecc = atmel_pmecc_create_user(nc->pmecc, &req); + if (IS_ERR(nand->pmecc)) + return PTR_ERR(nand->pmecc); + + chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.size = req.ecc.sectorsize; + chip->ecc.bytes = req.ecc.bytes / req.ecc.nsectors; + chip->ecc.strength = req.ecc.strength; + + chip->options |= NAND_NO_SUBPAGE_WRITE; + + mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + + return 0; +} + +static int atmel_nand_ecc_init(struct atmel_nand *nand) +{ + struct nand_chip *chip = &nand->base; + struct atmel_nand_controller *nc; + int ret; + + nc = to_nand_controller(chip->controller); + + switch (chip->ecc.mode) { + case NAND_ECC_NONE: + case NAND_ECC_SOFT: + /* + * Nothing to do, the core will initialize everything for us. + */ + break; + + case NAND_ECC_HW: + ret = atmel_nand_pmecc_init(chip); + if (ret) + return ret; + + chip->ecc.read_page = atmel_nand_pmecc_read_page; + chip->ecc.write_page = atmel_nand_pmecc_write_page; + chip->ecc.read_page_raw = atmel_nand_pmecc_read_page_raw; + chip->ecc.write_page_raw = atmel_nand_pmecc_write_page_raw; + break; + + default: + /* Other modes are not supported. */ + dev_err(nc->dev, "Unsupported ECC mode: %d\n", + chip->ecc.mode); + return -ENOTSUPP; + } + + return 0; +} + +static int atmel_hsmc_nand_ecc_init(struct atmel_nand *nand) +{ + struct nand_chip *chip = &nand->base; + int ret; + + ret = atmel_nand_ecc_init(nand); + if (ret) + return ret; + + if (chip->ecc.mode != NAND_ECC_HW) + return 0; + + /* Adjust the ECC operations for the HSMC IP. */ + chip->ecc.read_page = atmel_hsmc_nand_pmecc_read_page; + chip->ecc.write_page = atmel_hsmc_nand_pmecc_wri |