diff options
Diffstat (limited to 'drivers/dma')
-rw-r--r-- | drivers/dma/Kconfig | 15 | ||||
-rw-r--r-- | drivers/dma/Makefile | 1 | ||||
-rw-r--r-- | drivers/dma/bestcomm/sram.c | 1 | ||||
-rw-r--r-- | drivers/dma/edma.c | 5 | ||||
-rw-r--r-- | drivers/dma/fsldma.c | 2 | ||||
-rw-r--r-- | drivers/dma/imx-dma.c | 31 | ||||
-rw-r--r-- | drivers/dma/mmp_tdma.c | 7 | ||||
-rw-r--r-- | drivers/dma/mpc512x_dma.c | 2 | ||||
-rw-r--r-- | drivers/dma/ppc4xx/adma.c | 2 | ||||
-rw-r--r-- | drivers/dma/s3c24xx-dma.c | 1350 | ||||
-rw-r--r-- | drivers/dma/sh/rcar-hpbdma.c | 9 |
11 files changed, 1397 insertions, 28 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 526ec77c7ba0..c61a6eccf169 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -154,6 +154,18 @@ config TEGRA20_APB_DMA This DMA controller transfers data from memory to peripheral fifo or vice versa. It does not support memory to memory data transfer. +config S3C24XX_DMAC + tristate "Samsung S3C24XX DMA support" + depends on ARCH_S3C24XX && !S3C24XX_DMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Samsung S3C24XX DMA controller driver. The + DMA controller is having multiple DMA channels which can be + configured for different peripherals like audio, UART, SPI. + The DMA controller can transfer data from memory to peripheral, + periphal to memory, periphal to periphal and memory to memory. + source "drivers/dma/sh/Kconfig" config COH901318 @@ -195,9 +207,10 @@ config SIRF_DMA config TI_EDMA bool "TI EDMA support" - depends on ARCH_DAVINCI || ARCH_OMAP + depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE select DMA_ENGINE select DMA_VIRTUAL_CHANNELS + select TI_PRIV_EDMA default n help Enable support for the TI EDMA controller. This DMA diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index db89035b3626..0ce2da97e429 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -30,6 +30,7 @@ obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o +obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_PL330_DMA) += pl330.o obj-$(CONFIG_PCH_DMA) += pch_dma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o diff --git a/drivers/dma/bestcomm/sram.c b/drivers/dma/bestcomm/sram.c index 5e2ed30ba2c4..2074e0e3fa21 100644 --- a/drivers/dma/bestcomm/sram.c +++ b/drivers/dma/bestcomm/sram.c @@ -19,6 +19,7 @@ #include <linux/string.h> #include <linux/ioport.h> #include <linux/of.h> +#include <linux/of_address.h> #include <asm/io.h> #include <asm/mmu.h> diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index fd5e48c29746..bef8a368c8dd 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -305,7 +305,9 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( edma_alloc_slot(EDMA_CTLR(echan->ch_num), EDMA_SLOT_ANY); if (echan->slot[i] < 0) { + kfree(edesc); dev_err(dev, "Failed to allocate slot\n"); + kfree(edesc); return NULL; } } @@ -345,6 +347,7 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( ccnt = sg_dma_len(sg) / (acnt * bcnt); if (ccnt > (SZ_64K - 1)) { dev_err(dev, "Exceeded max SG segment size\n"); + kfree(edesc); return NULL; } cidx = acnt * bcnt; @@ -751,6 +754,6 @@ static void __exit edma_exit(void) } module_exit(edma_exit); -MODULE_AUTHOR("Matt Porter <mporter@ti.com>"); +MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>"); MODULE_DESCRIPTION("TI EDMA DMA engine driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index b3f3e90054f2..61517dd0d0b7 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -33,6 +33,8 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include "dmaengine.h" diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 78f8ca5fccee..55852c026791 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -437,17 +437,18 @@ static void dma_irq_handle_channel(struct imxdma_channel *imxdmac) struct imxdma_engine *imxdma = imxdmac->imxdma; int chno = imxdmac->channel; struct imxdma_desc *desc; + unsigned long flags; - spin_lock(&imxdma->lock); + spin_lock_irqsave(&imxdma->lock, flags); if (list_empty(&imxdmac->ld_active)) { - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); goto out; } desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node); - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); if (desc->sg) { u32 tmp; @@ -519,7 +520,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) { struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan); struct imxdma_engine *imxdma = imxdmac->imxdma; - unsigned long flags; int slot = -1; int i; @@ -527,7 +527,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) switch (d->type) { case IMXDMA_DESC_INTERLEAVED: /* Try to get a free 2D slot */ - spin_lock_irqsave(&imxdma->lock, flags); for (i = 0; i < IMX_DMA_2D_SLOTS; i++) { if ((imxdma->slots_2d[i].count > 0) && ((imxdma->slots_2d[i].xsr != d->x) || @@ -537,10 +536,8 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) slot = i; break; } - if (slot < 0) { - spin_unlock_irqrestore(&imxdma->lock, flags); + if (slot < 0) return -EBUSY; - } imxdma->slots_2d[slot].xsr = d->x; imxdma->slots_2d[slot].ysr = d->y; @@ -549,7 +546,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) imxdmac->slot_2d = slot; imxdmac->enabled_2d = true; - spin_unlock_irqrestore(&imxdma->lock, flags); if (slot == IMX_DMA_2D_SLOT_A) { d->config_mem &= ~CCR_MSEL_B; @@ -625,18 +621,17 @@ static void imxdma_tasklet(unsigned long data) struct imxdma_channel *imxdmac = (void *)data; struct imxdma_engine *imxdma = imxdmac->imxdma; struct imxdma_desc *desc; + unsigned long flags; - spin_lock(&imxdma->lock); + spin_lock_irqsave(&imxdma->lock, flags); if (list_empty(&imxdmac->ld_active)) { /* Someone might have called terminate all */ - goto out; + spin_unlock_irqrestore(&imxdma->lock, flags); + return; } desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node); - if (desc->desc.callback) - desc->desc.callback(desc->desc.callback_param); - /* If we are dealing with a cyclic descriptor, keep it on ld_active * and dont mark the descriptor as complete. * Only in non-cyclic cases it would be marked as complete @@ -663,7 +658,11 @@ static void imxdma_tasklet(unsigned long data) __func__, imxdmac->channel); } out: - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); + + if (desc->desc.callback) + desc->desc.callback(desc->desc.callback_param); + } static int imxdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, @@ -883,7 +882,7 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_cyclic( kfree(imxdmac->sg_list); imxdmac->sg_list = kcalloc(periods + 1, - sizeof(struct scatterlist), GFP_KERNEL); + sizeof(struct scatterlist), GFP_ATOMIC); if (!imxdmac->sg_list) return NULL; diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index 38cb517fb2eb..d3b6358e5a27 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -350,12 +350,7 @@ struct mmp_tdma_desc *mmp_tdma_alloc_descriptor(struct mmp_tdma_chan *tdmac) if (!gpool) return NULL; - tdmac->desc_arr = (void *)gen_pool_alloc(gpool, size); - if (!tdmac->desc_arr) - return NULL; - - tdmac->desc_arr_phys = gen_pool_virt_to_phys(gpool, - (unsigned long)tdmac->desc_arr); + tdmac->desc_arr = gen_pool_dma_alloc(gpool, size, &tdmac->desc_arr_phys); return tdmac->desc_arr; } diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index 2fe435377333..448750da4402 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -39,7 +39,9 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/of_address.h> #include <linux/of_device.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/random.h> diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 370ff8265630..e24b5ef486b5 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -42,6 +42,8 @@ #include <linux/uaccess.h> #include <linux/proc_fs.h> #include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <asm/dcr.h> #include <asm/dcr-regs.h> diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c new file mode 100644 index 000000000000..4cb127978636 --- /dev/null +++ b/drivers/dma/s3c24xx-dma.c @@ -0,0 +1,1350 @@ +/* + * S3C24XX DMA handling + * + * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de> + * + * based on amba-pl08x.c + * + * Copyright (c) 2006 ARM Ltd. + * Copyright (c) 2010 ST-Ericsson SA + * + * Author: Peter Pearse <peter.pearse@arm.com> + * Author: Linus Walleij <linus.walleij@stericsson.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals + * that can be routed to any of the 4 to 8 hardware-channels. + * + * Therefore on these DMA controllers the number of channels + * and the number of incoming DMA signals are two totally different things. + * It is usually not possible to theoretically handle all physical signals, + * so a multiplexing scheme with possible denial of use is necessary. + * + * Open items: + * - bursts + */ + +#include <linux/platform_device.h> +#include <linux/types.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/platform_data/dma-s3c24xx.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define MAX_DMA_CHANNELS 8 + +#define S3C24XX_DISRC 0x00 +#define S3C24XX_DISRCC 0x04 +#define S3C24XX_DISRCC_INC_INCREMENT 0 +#define S3C24XX_DISRCC_INC_FIXED BIT(0) +#define S3C24XX_DISRCC_LOC_AHB 0 +#define S3C24XX_DISRCC_LOC_APB BIT(1) + +#define S3C24XX_DIDST 0x08 +#define S3C24XX_DIDSTC 0x0c +#define S3C24XX_DIDSTC_INC_INCREMENT 0 +#define S3C24XX_DIDSTC_INC_FIXED BIT(0) +#define S3C24XX_DIDSTC_LOC_AHB 0 +#define S3C24XX_DIDSTC_LOC_APB BIT(1) +#define S3C24XX_DIDSTC_INT_TC0 0 +#define S3C24XX_DIDSTC_INT_RELOAD BIT(2) + +#define S3C24XX_DCON 0x10 + +#define S3C24XX_DCON_TC_MASK 0xfffff +#define S3C24XX_DCON_DSZ_BYTE (0 << 20) +#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20) +#define S3C24XX_DCON_DSZ_WORD (2 << 20) +#define S3C24XX_DCON_DSZ_MASK (3 << 20) +#define S3C24XX_DCON_DSZ_SHIFT 20 +#define S3C24XX_DCON_AUTORELOAD 0 +#define S3C24XX_DCON_NORELOAD BIT(22) +#define S3C24XX_DCON_HWTRIG BIT(23) +#define S3C24XX_DCON_HWSRC_SHIFT 24 +#define S3C24XX_DCON_SERV_SINGLE 0 +#define S3C24XX_DCON_SERV_WHOLE BIT(27) +#define S3C24XX_DCON_TSZ_UNIT 0 +#define S3C24XX_DCON_TSZ_BURST4 BIT(28) +#define S3C24XX_DCON_INT BIT(29) +#define S3C24XX_DCON_SYNC_PCLK 0 +#define S3C24XX_DCON_SYNC_HCLK BIT(30) +#define S3C24XX_DCON_DEMAND 0 +#define S3C24XX_DCON_HANDSHAKE BIT(31) + +#define S3C24XX_DSTAT 0x14 +#define S3C24XX_DSTAT_STAT_BUSY BIT(20) +#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff + +#define S3C24XX_DMASKTRIG 0x20 +#define S3C24XX_DMASKTRIG_SWTRIG BIT(0) +#define S3C24XX_DMASKTRIG_ON BIT(1) +#define S3C24XX_DMASKTRIG_STOP BIT(2) + +#define S3C24XX_DMAREQSEL 0x24 +#define S3C24XX_DMAREQSEL_HW BIT(0) + +/* + * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel + * for a DMA source. Instead only specific channels are valid. + * All of these SoCs have 4 physical channels and the number of request + * source bits is 3. Additionally we also need 1 bit to mark the channel + * as valid. + * Therefore we separate the chansel element of the channel data into 4 + * parts of 4 bits each, to hold the information if the channel is valid + * and the hw request source to use. + * + * Example: + * SDI is valid on channels 0, 2 and 3 - with varying hw request sources. + * For it the chansel field would look like + * + * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1 + * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2 + * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2 + */ +#define S3C24XX_CHANSEL_WIDTH 4 +#define S3C24XX_CHANSEL_VALID BIT(3) +#define S3C24XX_CHANSEL_REQ_MASK 7 + +/* + * struct soc_data - vendor-specific config parameters for individual SoCs + * @stride: spacing between the registers of each channel + * @has_reqsel: does the controller use the newer requestselection mechanism + * @has_clocks: are controllable dma-clocks present + */ +struct soc_data { + int stride; + bool has_reqsel; + bool has_clocks; +}; + +/* + * enum s3c24xx_dma_chan_state - holds the virtual channel states + * @S3C24XX_DMA_CHAN_IDLE: the channel is idle + * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport + * channel and is running a transfer on it + * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport + * channel to become available (only pertains to memcpy channels) + */ +enum s3c24xx_dma_chan_state { + S3C24XX_DMA_CHAN_IDLE, + S3C24XX_DMA_CHAN_RUNNING, + S3C24XX_DMA_CHAN_WAITING, +}; + +/* + * struct s3c24xx_sg - structure containing data per sg + * @src_addr: src address of sg + * @dst_addr: dst address of sg + * @len: transfer len in bytes + * @node: node for txd's dsg_list + */ +struct s3c24xx_sg { + dma_addr_t src_addr; + dma_addr_t dst_addr; + size_t len; + struct list_head node; +}; + +/* + * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor + * @vd: virtual DMA descriptor + * @dsg_list: list of children sg's + * @at: sg currently being transfered + * @width: transfer width + * @disrcc: value for source control register + * @didstc: value for destination control register + * @dcon: base value for dcon register + */ +struct s3c24xx_txd { + struct virt_dma_desc vd; + struct list_head dsg_list; + struct list_head *at; + u8 width; + u32 disrcc; + u32 didstc; + u32 dcon; +}; + +struct s3c24xx_dma_chan; + +/* + * struct s3c24xx_dma_phy - holder for the physical channels + * @id: physical index to this channel + * @valid: does the channel have all required elements + * @base: virtual memory base (remapped) for the this channel + * @irq: interrupt for this channel + * @clk: clock for this channel + * @lock: a lock to use when altering an instance of this struct + * @serving: virtual channel currently being served by this physicalchannel + * @host: a pointer to the host (internal use) + */ +struct s3c24xx_dma_phy { + unsigned int id; + bool valid; + void __iomem *base; + unsigned int irq; + struct clk *clk; + spinlock_t lock; + struct s3c24xx_dma_chan *serving; + struct s3c24xx_dma_engine *host; +}; + +/* + * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel + * @id: the id of the channel + * @name: name of the channel + * @vc: wrappped virtual channel + * @phy: the physical channel utilized by this channel, if there is one + * @runtime_addr: address for RX/TX according to the runtime config + * @at: active transaction on this channel + * @lock: a lock for this channel data + * @host: a pointer to the host (internal use) + * @state: whether the channel is idle, running etc + * @slave: whether this channel is a device (slave) or for memcpy + */ +struct s3c24xx_dma_chan { + int id; + const char *name; + struct virt_dma_chan vc; + struct s3c24xx_dma_phy *phy; + struct dma_slave_config cfg; + struct s3c24xx_txd *at; + struct s3c24xx_dma_engine *host; + enum s3c24xx_dma_chan_state state; + bool slave; +}; + +/* + * struct s3c24xx_dma_engine - the local state holder for the S3C24XX + * @pdev: the corresponding platform device + * @pdata: platform data passed in from the platform/machine + * @base: virtual memory base (remapped) + * @slave: slave engine for this instance + * @memcpy: memcpy engine for this instance + * @phy_chans: array of data for the physical channels + */ +struct s3c24xx_dma_engine { + struct platform_device *pdev; + const struct s3c24xx_dma_platdata *pdata; + struct soc_data *sdata; + void __iomem *base; + struct dma_device slave; + struct dma_device memcpy; + struct s3c24xx_dma_phy *phy_chans; +}; + +/* + * Physical channel handling + */ + +/* + * Check whether a certain channel is busy or not. + */ +static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy) +{ + unsigned int val = readl(phy->base + S3C24XX_DSTAT); + return val & S3C24XX_DSTAT_STAT_BUSY; +} + +static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan, + struct s3c24xx_dma_phy *phy) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; + int phyvalid; + + /* every phy is valid for memcopy channels */ + if (!s3cchan->slave) + return true; + + /* On newer variants all phys can be used for all virtual channels */ + if (s3cdma->sdata->has_reqsel) + return true; + + phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH)); + return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false; +} + +/* + * Allocate a physical channel for a virtual channel + * + * Try to locate a physical channel to be used for this transfer. If all + * are taken return NULL and the requester will have to cope by using + * some fallback PIO mode or retrying later. + */ +static +struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata; + struct s3c24xx_dma_phy *phy = NULL; + unsigned long flags; + int i; + int ret; + + if (s3cchan->slave) + cdata = &pdata->channels[s3cchan->id]; + + for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { + phy = &s3cdma->phy_chans[i]; + + if (!phy->valid) + continue; + + if (!s3c24xx_dma_phy_valid(s3cchan, phy)) + continue; + + spin_lock_irqsave(&phy->lock, flags); + + if (!phy->serving) { + phy->serving = s3cchan; + spin_unlock_irqrestore(&phy->lock, flags); + break; + } + + spin_unlock_irqrestore(&phy->lock, flags); + } + + /* No physical channel available, cope with it */ + if (i == s3cdma->pdata->num_phy_channels) { + dev_warn(&s3cdma->pdev->dev, "no phy channel available\n"); + return NULL; + } + + /* start the phy clock */ + if (s3cdma->sdata->has_clocks) { + ret = clk_enable(phy->clk); + if (ret) { + dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n", + phy->id, ret); + phy->serving = NULL; + return NULL; + } + } + + return phy; +} + +/* + * Mark the physical channel as free. + * + * This drops the link between the physical and virtual channel. + */ +static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy) +{ + struct s3c24xx_dma_engine *s3cdma = phy->host; + + if (s3cdma->sdata->has_clocks) + clk_disable(phy->clk); + + phy->serving = NULL; +} + +/* + * Stops the channel by writing the stop bit. + * This should not be used for an on-going transfer, but as a method of + * shutting down a channel (eg, when it's no longer used) or terminating a + * transfer. + */ +static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy) +{ + writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG); +} + +/* + * Virtual channel handling + */ + +static inline +struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct s3c24xx_dma_chan, vc.chan); +} + +static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_phy *phy = s3cchan->phy; + struct s3c24xx_txd *txd = s3cchan->at; + u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK; + + return tc * txd->width; +} + +static int s3c24xx_dma_set_runtime_config(struct s3c24xx_dma_chan *s3cchan, + struct dma_slave_config *config) +{ + if (!s3cchan->slave) + return -EINVAL; + + /* Reject definitely invalid configurations */ + if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || + config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) + return -EINVAL; + + s3cchan->cfg = *config; + + return 0; +} + +/* + * Transfer handling + */ + +static inline +struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct s3c24xx_txd, vd.tx); +} + +static struct s3c24xx_txd *s3c24xx_dma_get_txd(void) +{ + struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); + + if (txd) { + INIT_LIST_HEAD(&txd->dsg_list); + txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD; + } + + return txd; +} + +static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd) +{ + struct s3c24xx_sg *dsg, *_dsg; + + list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { + list_del(&dsg->node); + kfree(dsg); + } + + kfree(txd); +} + +static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan, + struct s3c24xx_txd *txd) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_phy *phy = s3cchan->phy; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node); + u32 dcon = txd->dcon; + u32 val; + + /* transfer-size and -count from len and width */ + switch (txd->width) { + case 1: + dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len; + break; + case 2: + dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2); + break; + case 4: + dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4); + break; + } + + if (s3cchan->slave) { + struct s3c24xx_dma_channel *cdata = + &pdata->channels[s3cchan->id]; + + if (s3cdma->sdata->has_reqsel) { + writel_relaxed((cdata->chansel << 1) | + S3C24XX_DMAREQSEL_HW, + phy->base + S3C24XX_DMAREQSEL); + } else { + int csel = cdata->chansel >> (phy->id * + S3C24XX_CHANSEL_WIDTH); + + csel &= S3C24XX_CHANSEL_REQ_MASK; + dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT; + dcon |= S3C24XX_DCON_HWTRIG; + } + } else { + if (s3cdma->sdata->has_reqsel) + writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL); + } + + writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC); + writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC); + writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST); + writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC); + writel_relaxed(dcon, phy->base + S3C24XX_DCON); + + val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG); + val &= ~S3C24XX_DMASKTRIG_STOP; + val |= S3C24XX_DMASKTRIG_ON; + + /* trigger the dma operation for memcpy transfers */ + if (!s3cchan->slave) + val |= S3C24XX_DMASKTRIG_SWTRIG; + + writel(val, phy->base + S3C24XX_DMASKTRIG); +} + +/* + * Set the initial DMA register values and start first sg. + */ +static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_phy *phy = s3cchan->phy; + struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc); + struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); + + list_del(&txd->vd.node); + + s3cchan->at = txd; + + /* Wait for channel inactive */ + while (s3c24xx_dma_phy_busy(phy)) + cpu_relax(); + + /* point to the first element of the sg list */ + txd->at = txd->dsg_list.next; + s3c24xx_dma_start_next_sg(s3cchan, txd); +} + +static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma, + struct s3c24xx_dma_chan *s3cchan) +{ + LIST_HEAD(head); + + vchan_get_all_descriptors(&s3cchan->vc, &head); + vchan_dma_desc_free_list(&s3cchan->vc, &head); +} + +/* + * Try to allocate a physical channel. When successful, assign it to + * this virtual channel, and initiate the next descriptor. The + * virtual channel lock must be held at this point. + */ +static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_phy *phy; + + phy = s3c24xx_dma_get_phy(s3cchan); + if (!phy) { + dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n", + s3cchan->name); + s3cchan->state = S3C24XX_DMA_CHAN_WAITING; + return; + } + + dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n", + phy->id, s3cchan->name); + + s3cchan->phy = phy; + s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; + + s3c24xx_dma_start_next_txd(s3cchan); +} + +static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy, + struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + + dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n", + phy->id, s3cchan->name); + + /* + * We do this without taking the lock; we're really only concerned + * about whether this pointer is NULL or not, and we're guaranteed + * that this will only be called when it _already_ is non-NULL. + */ + phy->serving = s3cchan; + s3cchan->phy = phy; + s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; + s3c24xx_dma_start_next_txd(s3cchan); +} + +/* + * Free a physical DMA channel, potentially reallocating it to another + * virtual channel if we have any pending. + */ +static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_chan *p, *next; + +retry: + next = NULL; + + /* Find a waiting virtual channel for the next transfer. */ + list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node) + if (p->state == S3C24XX_DMA_CHAN_WAITING) { + next = p; + break; + } + + if (!next) { + list_for_each_entry(p, &s3cdma->slave.channels, + vc.chan.device_node) + if (p->state == S3C24XX_DMA_CHAN_WAITING && + s3c24xx_dma_phy_valid(p, s3cchan->phy)) { + next = p; + break; + } + } + + /* Ensure that the physical channel is stopped */ + s3c24xx_dma_terminate_phy(s3cchan->phy); + + if (next) { + bool success; + + /* + * Eww. We know this isn't going to deadlock + * but lockdep probably doesn't. + */ + spin_lock(&next->vc.lock); + /* Re-check the state now that we have the lock */ + success = next->state == S3C24XX_DMA_CHAN_WAITING; + if (success) + s3c24xx_dma_phy_reassign_start(s3cchan->phy, next); + spin_unlock(&next->vc.lock); + + /* If the state changed, try to find another channel */ + if (!success) + goto retry; + } else { + /* No more jobs, so free up the physical channel */ + s3c24xx_dma_put_phy(s3cchan->phy); + } + + s3cchan->phy = NULL; + s3cchan->state = S3C24XX_DMA_CHAN_IDLE; +} + +static void s3c24xx_dma_unmap_buffers(struct s3c24xx_txd *txd) +{ + struct device *dev = txd->vd.tx.chan->device->dev; + struct s3c24xx_sg *dsg; + + if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + if (txd->vd.tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE) + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_single(dev, dsg->src_addr, dsg->len, + DMA_TO_DEVICE); + else { + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_page(dev, dsg->src_addr, dsg->len, + DMA_TO_DEVICE); + } + } + + if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + if (txd->vd.tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE) + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_single(dev, dsg->dst_addr, dsg->len, + DMA_FROM_DEVICE); + else + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_page(dev, dsg->dst_addr, dsg->len, + DMA_FROM_DEVICE); + } +} + +static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd) +{ + struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan); + + if (!s3cchan->slave) + s3c24xx_dma_unmap_buffers(txd); + + s3c24xx_dma_free_txd(txd); +} + +static irqreturn_t s3c24xx_dma_irq(int irq, void *data) +{ + struct s3c24xx_dma_phy *phy = data; + struct s3c24xx_dma_chan *s3cchan = phy->serving; + struct s3c24xx_txd *txd; + + dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id); + + /* + * Interrupts happen to notify the completion of a transfer and the + * channel should have moved into its stop state already on its own. + * Therefore interrupts on channels not bound to a virtual channel + * should never happen. Nevertheless send a terminate command to the + * channel if the unlikely case happens. + */ + if (unlikely(!s3cchan)) { + dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n", + phy->id); + + s3c24xx_dma_terminate_phy(phy); + + return IRQ_HANDLED; + } + + spin_lock(&s3cchan->vc.lock); + txd = s3cchan->at; + if (txd) { + /* when more sg's are in this txd, start the next one */ + if (!list_is_last(txd->at, &txd->dsg_list)) { + txd->at = txd->at->next; + s3c24xx_dma_start_next_sg(s3cchan, txd); + } else { + s3cchan->at = NULL; + vchan_cookie_complete(&txd->vd); + + /* + * And start the next descriptor (if any), + * otherwise free this channel. + */ + if (vchan_next_desc(&s3cchan->vc)) + s3c24xx_dma_start_next_txd(s3cchan); + else + s3c24xx_dma_phy_free(s3cchan); + } + } + spin_unlock(&s3cchan->vc.lock); + + return IRQ_HANDLED; +} + +/* + * The DMA ENGINE API + */ + +static int s3c24xx_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&s3cchan->vc.lock, flags); + + switch (cmd) { + case DMA_SLAVE_CONFIG: + ret = s3c24xx_dma_set_runtime_config(s3cchan, + (struct dma_slave_config *)arg); + break; + case DMA_TERMINATE_ALL: + if (!s3cchan->phy && !s3cchan->at) { + dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n", + s3cchan->id); + ret = -EINVAL; + break; + } + + s3cchan->state = S3C24XX_DMA_CHAN_IDLE; + + /* Mark physical channel as free */ + if (s3cchan->phy) + s3c24xx_dma_phy_free(s3cchan); + + /* Dequeue current job */ + if (s3cchan->at) { + s3c24xx_dma_desc_free(&s3cchan->at->vd); + s3cchan->at = NULL; + } + + /* Dequeue jobs not yet fired as well */ + s3c24xx_dma_free_txd_list(s3cdma, s3cchan); + break; + default: + /* Unknown command */ + ret = -ENXIO; + break; + } + + spin_unlock_irqrestore(&s3cchan->vc.lock, flags |