diff options
author | Vinod Koul <vinod.koul@intel.com> | 2017-11-14 10:34:56 +0530 |
---|---|---|
committer | Vinod Koul <vinod.koul@intel.com> | 2017-11-14 10:34:56 +0530 |
commit | 9c6027133626efd773fa84478c5d8c9209df41f6 (patch) | |
tree | 33974ace800bee72c96cf32b05aaa019034728af /drivers/dma | |
parent | 2c8528592c3280c3e2dc85a804120ca64137de32 (diff) | |
parent | d83f4131c2e06ef93cd3d6d44bb3475728790ab6 (diff) |
Merge branch 'topic/stm' into for-linus
Diffstat (limited to 'drivers/dma')
-rw-r--r-- | drivers/dma/Kconfig | 21 | ||||
-rw-r--r-- | drivers/dma/Makefile | 2 | ||||
-rw-r--r-- | drivers/dma/stm32-dmamux.c | 327 | ||||
-rw-r--r-- | drivers/dma/stm32-mdma.c | 1682 |
4 files changed, 2032 insertions, 0 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 48cf8df7255f..4002d1c65221 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -483,6 +483,27 @@ config STM32_DMA If you have a board based on such a MCU and wish to use DMA say Y here. +config STM32_DMAMUX + bool "STMicroelectronics STM32 dma multiplexer support" + depends on STM32_DMA || COMPILE_TEST + help + Enable support for the on-chip DMA multiplexer on STMicroelectronics + STM32 MCUs. + If you have a board based on such a MCU and wish to use DMAMUX say Y + here. + +config STM32_MDMA + bool "STMicroelectronics STM32 master dma support" + depends on ARCH_STM32 || COMPILE_TEST + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip MDMA controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC and wish to use the master DMA + say Y here. + config S3C24XX_DMAC bool "Samsung S3C24XX DMA support" depends on ARCH_S3C24XX || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index f08f8de1b567..4d2376fdea01 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -59,6 +59,8 @@ obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_STM32_DMA) += stm32-dma.o +obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o +obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c new file mode 100644 index 000000000000..d5db0f6e1ff8 --- /dev/null +++ b/drivers/dma/stm32-dmamux.c @@ -0,0 +1,327 @@ +/* + * + * Copyright (C) STMicroelectronics SA 2017 + * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com> + * Pierre-Yves Mordret <pierre-yves.mordret@st.com> + * + * License terms: GPL V2.0. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * DMA Router driver for STM32 DMA MUX + * + * Based on TI DMA Crossbar driver + * + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/reset.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#define STM32_DMAMUX_CCR(x) (0x4 * (x)) +#define STM32_DMAMUX_MAX_DMA_REQUESTS 32 +#define STM32_DMAMUX_MAX_REQUESTS 255 + +struct stm32_dmamux { + u32 master; + u32 request; + u32 chan_id; +}; + +struct stm32_dmamux_data { + struct dma_router dmarouter; + struct clk *clk; + struct reset_control *rst; + void __iomem *iomem; + u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ + u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ + spinlock_t lock; /* Protects register access */ + unsigned long *dma_inuse; /* Used DMA channel */ + u32 dma_reqs[]; /* Number of DMA Request per DMA masters. + * [0] holds number of DMA Masters. + * To be kept at very end end of this structure + */ +}; + +static inline u32 stm32_dmamux_read(void __iomem *iomem, u32 reg) +{ + return readl_relaxed(iomem + reg); +} + +static inline void stm32_dmamux_write(void __iomem *iomem, u32 reg, u32 val) +{ + writel_relaxed(val, iomem + reg); +} + +static void stm32_dmamux_free(struct device *dev, void *route_data) +{ + struct stm32_dmamux_data *dmamux = dev_get_drvdata(dev); + struct stm32_dmamux *mux = route_data; + unsigned long flags; + + /* Clear dma request */ + spin_lock_irqsave(&dmamux->lock, flags); + + stm32_dmamux_write(dmamux->iomem, STM32_DMAMUX_CCR(mux->chan_id), 0); + clear_bit(mux->chan_id, dmamux->dma_inuse); + + if (!IS_ERR(dmamux->clk)) + clk_disable(dmamux->clk); + + spin_unlock_irqrestore(&dmamux->lock, flags); + + dev_dbg(dev, "Unmapping DMAMUX(%u) to DMA%u(%u)\n", + mux->request, mux->master, mux->chan_id); + + kfree(mux); +} + +static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct stm32_dmamux_data *dmamux = platform_get_drvdata(pdev); + struct stm32_dmamux *mux; + u32 i, min, max; + int ret; + unsigned long flags; + + if (dma_spec->args_count != 3) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + if (dma_spec->args[0] > dmamux->dmamux_requests) { + dev_err(&pdev->dev, "invalid mux request number: %d\n", + dma_spec->args[0]); + return ERR_PTR(-EINVAL); + } + + mux = kzalloc(sizeof(*mux), GFP_KERNEL); + if (!mux) + return ERR_PTR(-ENOMEM); + + spin_lock_irqsave(&dmamux->lock, flags); + mux->chan_id = find_first_zero_bit(dmamux->dma_inuse, + dmamux->dma_requests); + set_bit(mux->chan_id, dmamux->dma_inuse); + spin_unlock_irqrestore(&dmamux->lock, flags); + + if (mux->chan_id == dmamux->dma_requests) { + dev_err(&pdev->dev, "Run out of free DMA requests\n"); + ret = -ENOMEM; + goto error; + } + + /* Look for DMA Master */ + for (i = 1, min = 0, max = dmamux->dma_reqs[i]; + i <= dmamux->dma_reqs[0]; + min += dmamux->dma_reqs[i], max += dmamux->dma_reqs[++i]) + if (mux->chan_id < max) + break; + mux->master = i - 1; + + /* The of_node_put() will be done in of_dma_router_xlate function */ + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", i - 1); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + ret = -EINVAL; + goto error; + } + + /* Set dma request */ + spin_lock_irqsave(&dmamux->lock, flags); + if (!IS_ERR(dmamux->clk)) { + ret = clk_enable(dmamux->clk); + if (ret < 0) { + spin_unlock_irqrestore(&dmamux->lock, flags); + dev_err(&pdev->dev, "clk_prep_enable issue: %d\n", ret); + goto error; + } + } + spin_unlock_irqrestore(&dmamux->lock, flags); + + mux->request = dma_spec->args[0]; + + /* craft DMA spec */ + dma_spec->args[3] = dma_spec->args[2]; + dma_spec->args[2] = dma_spec->args[1]; + dma_spec->args[1] = 0; + dma_spec->args[0] = mux->chan_id - min; + dma_spec->args_count = 4; + + stm32_dmamux_write(dmamux->iomem, STM32_DMAMUX_CCR(mux->chan_id), + mux->request); + dev_dbg(&pdev->dev, "Mapping DMAMUX(%u) to DMA%u(%u)\n", + mux->request, mux->master, mux->chan_id); + + return mux; + +error: + clear_bit(mux->chan_id, dmamux->dma_inuse); + kfree(mux); + return ERR_PTR(ret); +} + +static const struct of_device_id stm32_stm32dma_master_match[] = { + { .compatible = "st,stm32-dma", }, + {}, +}; + +static int stm32_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; + struct device_node *dma_node; + struct stm32_dmamux_data *stm32_dmamux; + struct resource *res; + void __iomem *iomem; + int i, count, ret; + u32 dma_req; + + if (!node) + return -ENODEV; + + count = device_property_read_u32_array(&pdev->dev, "dma-masters", + NULL, 0); + if (count < 0) { + dev_err(&pdev->dev, "Can't get DMA master(s) node\n"); + return -ENODEV; + } + + stm32_dmamux = devm_kzalloc(&pdev->dev, sizeof(*stm32_dmamux) + + sizeof(u32) * (count + 1), GFP_KERNEL); + if (!stm32_dmamux) + return -ENOMEM; + + dma_req = 0; + for (i = 1; i <= count; i++) { + dma_node = of_parse_phandle(node, "dma-masters", i - 1); + + match = of_match_node(stm32_stm32dma_master_match, dma_node); + if (!match) { + dev_err(&pdev->dev, "DMA master is not supported\n"); + of_node_put(dma_node); + return -EINVAL; + } + + if (of_property_read_u32(dma_node, "dma-requests", + &stm32_dmamux->dma_reqs[i])) { + dev_info(&pdev->dev, + "Missing MUX output information, using %u.\n", + STM32_DMAMUX_MAX_DMA_REQUESTS); + stm32_dmamux->dma_reqs[i] = + STM32_DMAMUX_MAX_DMA_REQUESTS; + } + dma_req += stm32_dmamux->dma_reqs[i]; + of_node_put(dma_node); + } + + if (dma_req > STM32_DMAMUX_MAX_DMA_REQUESTS) { + dev_err(&pdev->dev, "Too many DMA Master Requests to manage\n"); + return -ENODEV; + } + + stm32_dmamux->dma_requests = dma_req; + stm32_dmamux->dma_reqs[0] = count; + stm32_dmamux->dma_inuse = devm_kcalloc(&pdev->dev, + BITS_TO_LONGS(dma_req), + sizeof(unsigned long), + GFP_KERNEL); + if (!stm32_dmamux->dma_inuse) + return -ENOMEM; + + if (device_property_read_u32(&pdev->dev, "dma-requests", + &stm32_dmamux->dmamux_requests)) { + stm32_dmamux->dmamux_requests = STM32_DMAMUX_MAX_REQUESTS; + dev_warn(&pdev->dev, "DMAMUX defaulting on %u requests\n", + stm32_dmamux->dmamux_requests); + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + + iomem = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(iomem)) + return PTR_ERR(iomem); + + spin_lock_init(&stm32_dmamux->lock); + + stm32_dmamux->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(stm32_dmamux->clk)) { + ret = PTR_ERR(stm32_dmamux->clk); + if (ret == -EPROBE_DEFER) + dev_info(&pdev->dev, "Missing controller clock\n"); + return ret; + } + + stm32_dmamux->rst = devm_reset_control_get(&pdev->dev, NULL); + if (!IS_ERR(stm32_dmamux->rst)) { + reset_control_assert(stm32_dmamux->rst); + udelay(2); + reset_control_deassert(stm32_dmamux->rst); + } + + stm32_dmamux->iomem = iomem; + stm32_dmamux->dmarouter.dev = &pdev->dev; + stm32_dmamux->dmarouter.route_free = stm32_dmamux_free; + + platform_set_drvdata(pdev, stm32_dmamux); + + if (!IS_ERR(stm32_dmamux->clk)) { + ret = clk_prepare_enable(stm32_dmamux->clk); + if (ret < 0) { + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); + return ret; + } + } + + /* Reset the dmamux */ + for (i = 0; i < stm32_dmamux->dma_requests; i++) + stm32_dmamux_write(stm32_dmamux->iomem, STM32_DMAMUX_CCR(i), 0); + + if (!IS_ERR(stm32_dmamux->clk)) + clk_disable(stm32_dmamux->clk); + + return of_dma_router_register(node, stm32_dmamux_route_allocate, + &stm32_dmamux->dmarouter); +} + +static const struct of_device_id stm32_dmamux_match[] = { + { .compatible = "st,stm32h7-dmamux" }, + {}, +}; + +static struct platform_driver stm32_dmamux_driver = { + .probe = stm32_dmamux_probe, + .driver = { + .name = "stm32-dmamux", + .of_match_table = stm32_dmamux_match, + }, +}; + +static int __init stm32_dmamux_init(void) +{ + return platform_driver_register(&stm32_dmamux_driver); +} +arch_initcall(stm32_dmamux_init); + +MODULE_DESCRIPTION("DMA Router driver for STM32 DMA MUX"); +MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>"); +MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c new file mode 100644 index 000000000000..daa1602eb9f5 --- /dev/null +++ b/drivers/dma/stm32-mdma.c @@ -0,0 +1,1682 @@ +/* + * + * Copyright (C) STMicroelectronics SA 2017 + * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com> + * Pierre-Yves Mordret <pierre-yves.mordret@st.com> + * + * License terms: GPL V2.0. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * Driver for STM32 MDMA controller + * + * Inspired by stm32-dma.c and dma-jz4780.c + * + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/jiffies.h> +#include <linux/list.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include "virt-dma.h" + +/* MDMA Generic getter/setter */ +#define STM32_MDMA_SHIFT(n) (ffs(n) - 1) +#define STM32_MDMA_SET(n, mask) (((n) << STM32_MDMA_SHIFT(mask)) & \ + (mask)) +#define STM32_MDMA_GET(n, mask) (((n) & (mask)) >> \ + STM32_MDMA_SHIFT(mask)) + +#define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */ +#define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */ + +/* MDMA Channel x interrupt/status register */ +#define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */ +#define STM32_MDMA_CISR_CRQA BIT(16) +#define STM32_MDMA_CISR_TCIF BIT(4) +#define STM32_MDMA_CISR_BTIF BIT(3) +#define STM32_MDMA_CISR_BRTIF BIT(2) +#define STM32_MDMA_CISR_CTCIF BIT(1) +#define STM32_MDMA_CISR_TEIF BIT(0) + +/* MDMA Channel x interrupt flag clear register */ +#define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x)) +#define STM32_MDMA_CIFCR_CLTCIF BIT(4) +#define STM32_MDMA_CIFCR_CBTIF BIT(3) +#define STM32_MDMA_CIFCR_CBRTIF BIT(2) +#define STM32_MDMA_CIFCR_CCTCIF BIT(1) +#define STM32_MDMA_CIFCR_CTEIF BIT(0) +#define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \ + | STM32_MDMA_CIFCR_CBTIF \ + | STM32_MDMA_CIFCR_CBRTIF \ + | STM32_MDMA_CIFCR_CCTCIF \ + | STM32_MDMA_CIFCR_CTEIF) + +/* MDMA Channel x error status register */ +#define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x)) +#define STM32_MDMA_CESR_BSE BIT(11) +#define STM32_MDMA_CESR_ASR BIT(10) +#define STM32_MDMA_CESR_TEMD BIT(9) +#define STM32_MDMA_CESR_TELD BIT(8) +#define STM32_MDMA_CESR_TED BIT(7) +#define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0) + +/* MDMA Channel x control register */ +#define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x)) +#define STM32_MDMA_CCR_SWRQ BIT(16) +#define STM32_MDMA_CCR_WEX BIT(14) +#define STM32_MDMA_CCR_HEX BIT(13) +#define STM32_MDMA_CCR_BEX BIT(12) +#define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6) +#define STM32_MDMA_CCR_PL(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CCR_PL_MASK) +#define STM32_MDMA_CCR_TCIE BIT(5) +#define STM32_MDMA_CCR_BTIE BIT(4) +#define STM32_MDMA_CCR_BRTIE BIT(3) +#define STM32_MDMA_CCR_CTCIE BIT(2) +#define STM32_MDMA_CCR_TEIE BIT(1) +#define STM32_MDMA_CCR_EN BIT(0) +#define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \ + | STM32_MDMA_CCR_BTIE \ + | STM32_MDMA_CCR_BRTIE \ + | STM32_MDMA_CCR_CTCIE \ + | STM32_MDMA_CCR_TEIE) + +/* MDMA Channel x transfer configuration register */ +#define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x)) +#define STM32_MDMA_CTCR_BWM BIT(31) +#define STM32_MDMA_CTCR_SWRM BIT(30) +#define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28) +#define STM32_MDMA_CTCR_TRGM(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_TRGM_MSK) +#define STM32_MDMA_CTCR_TRGM_GET(n) STM32_MDMA_GET((n), \ + STM32_MDMA_CTCR_TRGM_MSK) +#define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26) +#define STM32_MDMA_CTCR_PAM(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTCR_PAM_MASK) +#define STM32_MDMA_CTCR_PKE BIT(25) +#define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18) +#define STM32_MDMA_CTCR_TLEN(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_TLEN_MSK) +#define STM32_MDMA_CTCR_TLEN_GET(n) STM32_MDMA_GET((n), \ + STM32_MDMA_CTCR_TLEN_MSK) +#define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18) +#define STM32_MDMA_CTCR_LEN2(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_LEN2_MSK) +#define STM32_MDMA_CTCR_LEN2_GET(n) STM32_MDMA_GET((n), \ + STM32_MDMA_CTCR_LEN2_MSK) +#define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15) +#define STM32_MDMA_CTCR_DBURST(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTCR_DBURST_MASK) +#define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12) +#define STM32_MDMA_CTCR_SBURST(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTCR_SBURST_MASK) +#define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10) +#define STM32_MDMA_CTCR_DINCOS(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_DINCOS_MASK) +#define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8) +#define STM32_MDMA_CTCR_SINCOS(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_SINCOS_MASK) +#define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6) +#define STM32_MDMA_CTCR_DSIZE(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTCR_DSIZE_MASK) +#define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4) +#define STM32_MDMA_CTCR_SSIZE(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTCR_SSIZE_MASK) +#define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2) +#define STM32_MDMA_CTCR_DINC(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_DINC_MASK) +#define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0) +#define STM32_MDMA_CTCR_SINC(n) STM32_MDMA_SET((n), \ + STM32_MDMA_CTCR_SINC_MASK) +#define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \ + | STM32_MDMA_CTCR_DINC_MASK \ + | STM32_MDMA_CTCR_SINCOS_MASK \ + | STM32_MDMA_CTCR_DINCOS_MASK \ + | STM32_MDMA_CTCR_LEN2_MSK \ + | STM32_MDMA_CTCR_TRGM_MSK) + +/* MDMA Channel x block number of data register */ +#define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x)) +#define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20) +#define STM32_MDMA_CBNDTR_BRC(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CBNDTR_BRC_MK) +#define STM32_MDMA_CBNDTR_BRC_GET(n) STM32_MDMA_GET((n), \ + STM32_MDMA_CBNDTR_BRC_MK) + +#define STM32_MDMA_CBNDTR_BRDUM BIT(19) +#define STM32_MDMA_CBNDTR_BRSUM BIT(18) +#define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0) +#define STM32_MDMA_CBNDTR_BNDT(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CBNDTR_BNDT_MASK) + +/* MDMA Channel x source address register */ +#define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x)) + +/* MDMA Channel x destination address register */ +#define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x)) + +/* MDMA Channel x block repeat address update register */ +#define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x)) +#define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16) +#define STM32_MDMA_CBRUR_DUV(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CBRUR_DUV_MASK) +#define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0) +#define STM32_MDMA_CBRUR_SUV(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CBRUR_SUV_MASK) + +/* MDMA Channel x link address register */ +#define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x)) + +/* MDMA Channel x trigger and bus selection register */ +#define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x)) +#define STM32_MDMA_CTBR_DBUS BIT(17) +#define STM32_MDMA_CTBR_SBUS BIT(16) +#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(7, 0) +#define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \ + STM32_MDMA_CTBR_TSEL_MASK) + +/* MDMA Channel x mask address register */ +#define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x)) + +/* MDMA Channel x mask data register */ +#define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x)) + +#define STM32_MDMA_MAX_BUF_LEN 128 +#define STM32_MDMA_MAX_BLOCK_LEN 65536 +#define STM32_MDMA_MAX_CHANNELS 63 +#define STM32_MDMA_MAX_REQUESTS 256 +#define STM32_MDMA_MAX_BURST 128 +#define STM32_MDMA_VERY_HIGH_PRIORITY 0x11 + +enum stm32_mdma_trigger_mode { + STM32_MDMA_BUFFER, + STM32_MDMA_BLOCK, + STM32_MDMA_BLOCK_REP, + STM32_MDMA_LINKED_LIST, +}; + +enum stm32_mdma_width { + STM32_MDMA_BYTE, + STM32_MDMA_HALF_WORD, + STM32_MDMA_WORD, + STM32_MDMA_DOUBLE_WORD, +}; + +enum stm32_mdma_inc_mode { + STM32_MDMA_FIXED = 0, + STM32_MDMA_INC = 2, + STM32_MDMA_DEC = 3, +}; + +struct stm32_mdma_chan_config { + u32 request; + u32 priority_level; + u32 transfer_config; + u32 mask_addr; + u32 mask_data; +}; + +struct stm32_mdma_hwdesc { + u32 ctcr; + u32 cbndtr; + u32 csar; + u32 cdar; + u32 cbrur; + u32 clar; + u32 ctbr; + u32 dummy; + u32 cmar; + u32 cmdr; +} __aligned(64); + +struct stm32_mdma_desc { + struct virt_dma_desc vdesc; + u32 ccr; + struct stm32_mdma_hwdesc *hwdesc; + dma_addr_t hwdesc_phys; + bool cyclic; + u32 count; +}; + +struct stm32_mdma_chan { + struct virt_dma_chan vchan; + struct dma_pool *desc_pool; + u32 id; + struct stm32_mdma_desc *desc; + u32 curr_hwdesc; + struct dma_slave_config dma_config; + struct stm32_mdma_chan_config chan_config; + bool busy; + u32 mem_burst; + u32 mem_width; +}; + +struct stm32_mdma_device { + struct dma_device ddev; + void __iomem *base; + struct clk *clk; + int irq; + struct reset_control *rst; + u32 nr_channels; + u32 nr_requests; + u32 nr_ahb_addr_masks; + struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS]; + u32 ahb_addr_masks[]; +}; + +static struct stm32_mdma_device *stm32_mdma_get_dev( + struct stm32_mdma_chan *chan) +{ + return container_of(chan->vchan.chan.device, struct stm32_mdma_device, + ddev); +} + +static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c) +{ + return container_of(c, struct stm32_mdma_chan, vchan.chan); +} + +static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct stm32_mdma_desc, vdesc); +} + +static struct device *chan2dev(struct stm32_mdma_chan *chan) +{ + return &chan->vchan.chan.dev->device; +} + +static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev) +{ + return mdma_dev->ddev.dev; +} + +static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg) +{ + return readl_relaxed(dmadev->base + reg); +} + +static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val) +{ + writel_relaxed(val, dmadev->base + reg); +} + +static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg, + u32 mask) +{ + void __iomem *addr = dmadev->base + reg; + + writel_relaxed(readl_relaxed(addr) | mask, addr); +} + +static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg, + u32 mask) +{ + void __iomem *addr = dmadev->base + reg; + + writel_relaxed(readl_relaxed(addr) & ~mask, addr); +} + +static struct stm32_mdma_desc *stm32_mdma_alloc_desc( + struct stm32_mdma_chan *chan, u32 count) +{ + struct stm32_mdma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->hwdesc = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, + &desc->hwdesc_phys); + if (!desc->hwdesc) { + dev_err(chan2dev(chan), "Failed to allocate descriptor\n"); + kfree(desc); + return NULL; + } + + desc->count = count; + + return desc; +} + +static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc) +{ + struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc); + struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan); + + dma_pool_free(chan->desc_pool, desc->hwdesc, desc->hwdesc_phys); + kfree(desc); +} + +static int stm32_mdma_get_width(struct stm32_mdma_chan *chan, + enum dma_slave_buswidth width) +{ + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + case DMA_SLAVE_BUSWIDTH_2_BYTES: + case DMA_SLAVE_BUSWIDTH_4_BYTES: + case DMA_SLAVE_BUSWIDTH_8_BYTES: + return ffs(width) - 1; + default: + dev_err(chan2dev(chan), "Dma bus width %i not supported\n", + width); + return -EINVAL; + } +} + +static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr, + u32 buf_len, u32 tlen) +{ + enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES; + + for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES; + max_width > DMA_SLAVE_BUSWIDTH_1_BYTE; + max_width >>= 1) { + /* + * Address and buffer length both have to be aligned on + * bus width + */ + if ((((buf_len | addr) & (max_width - 1)) == 0) && + tlen >= max_width) + break; + } + + return max_width; +} + +static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst, + enum dma_slave_buswidth width) +{ + u32 best_burst = max_burst; + u32 burst_len = best_burst * width; + + while ((burst_len > 0) && (tlen % burst_len)) { + best_burst = best_burst >> 1; + burst_len = best_burst * width; + } + + return (best_burst > 0) ? best_burst : 1; +} + +static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan) +{ + struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); + u32 ccr, cisr, id, reg; + int ret; + + id = chan->id; + reg = STM32_MDMA_CCR(id); + + /* Disable interrupts */ + stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK); + + ccr = stm32_mdma_read(dmadev, reg); + if (ccr & STM32_MDMA_CCR_EN) { + stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN); + + /* Ensure that any ongoing transfer has been completed */ + ret = readl_relaxed_poll_timeout_atomic( + dmadev->base + STM32_MDMA_CISR(id), cisr, + (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000); + if (ret) { + dev_err(chan2dev(chan), "%s: timeout!\n", __func__); + return -EBUSY; + } + } + + return 0; +} + +static void stm32_mdma_stop(struct stm32_mdma_chan *chan) +{ + struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); + u32 status; + int ret; + + /* Disable DMA */ + ret = stm32_mdma_disable_chan(chan); + if (ret < 0) + return; + + /* Clear interrupt status if it is there */ + status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); + if (status) { + dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n", + __func__, status); + stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status); + } + + chan->busy = false; +} + +static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr, + u32 ctbr_mask, u32 src_addr) +{ + u32 mask; + int i; + + /* Check if memory device is on AHB or AXI */ + *ctbr &= ~ctbr_mask; + mask = src_addr & 0xF0000000; + for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) { + if (mask == dmadev->ahb_addr_masks[i]) { + *ctbr |= ctbr_mask; + break; + } + } +} + +static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, + enum dma_transfer_direction direction, + u32 *mdma_ccr, u32 *mdma_ctcr, + u32 *mdma_ctbr, dma_addr_t addr, + u32 buf_len) +{ + struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; + enum dma_slave_buswidth src_addr_width, dst_addr_width; + phys_addr_t src_addr, dst_addr; + int src_bus_width, dst_bus_width; + u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst; + u32 ccr, ctcr, ctbr, tlen; + + src_addr_width = chan->dma_config.src_addr_width; + dst_addr_width = chan->dma_config.dst_addr_width; + src_maxburst = chan->dma_config.src_maxburst; + dst_maxburst = chan->dma_config.dst_maxburst; + + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); + ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); + ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); + + /* Enable HW request mode */ + ctcr &= ~STM32_MDMA_CTCR_SWRM; + + /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */ + ctcr &= ~STM32_MDMA_CTCR_CFG_MASK; + ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK; + + /* + * For buffer transfer length (TLEN) we have to set + * the number of bytes - 1 in CTCR register + */ + tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr); + ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK; + ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1)); + + /* Disable Pack Enable */ + ctcr &= ~STM32_MDMA_CTCR_PKE; + + /* Check burst size constraints */ + if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST || + dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) { + dev_err(chan2dev(chan), + "burst size * bus width higher than %d bytes\n", + STM32_MDMA_MAX_BURST); + return -EINVAL; + } + + if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) || + (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) { + dev_err(chan2dev(chan), "burst size must be a power of 2\n"); + return -EINVAL; + } + + /* + * Configure channel control: + * - Clear SW request as in this case this is a HW one + * - Clear WEX, HEX and BEX bits + * - Set priority level + */ + ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX | + STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK); + ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level); + + /* Configure Trigger selection */ + ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK; + ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request); + + switch (direction) { + case DMA_MEM_TO_DEV: + dst_addr = chan->dma_config.dst_addr; + + /* Set device data size */ + dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); + if (dst_bus_width < 0) + return dst_bus_width; + ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK; + ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width); + + /* Set device burst value */ + dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, + dst_maxburst, + dst_addr_width); + chan->mem_burst = dst_best_burst; + ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK; + ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst))); + + /* Set memory data size */ + src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen); + chan->mem_width = src_addr_width; + src_bus_width = stm32_mdma_get_width(chan, src_addr_width); + if (src_bus_width < 0) + return src_bus_width; + ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK | + STM32_MDMA_CTCR_SINCOS_MASK; + ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) | + STM32_MDMA_CTCR_SINCOS(src_bus_width); + + /* Set memory burst value */ + src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width; + src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, + src_maxburst, + src_addr_width); + chan->mem_burst = src_best_burst; + ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK; + ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst))); + + /* Select bus */ + stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, + dst_addr); + + if (dst_bus_width != src_bus_width) + ctcr |= STM32_MDMA_CTCR_PKE; + + /* Set destination address */ + stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr); + break; + + case DMA_DEV_TO_MEM: + src_addr = chan->dma_config.src_addr; + + /* Set device data size */ + src_bus_width = stm32_mdma_get_width(chan, src_addr_width); + if (src_bus_width < 0) + return src_bus_width; + ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK; + ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width); + + /* Set device burst value */ + src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, + src_maxburst, + src_addr_width); + ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK; + ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst))); + + /* Set memory data size */ + dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen); + chan->mem_width = dst_addr_width; + dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); + if (dst_bus_width < 0) + return dst_bus_width; + ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK | + STM32_MDMA_CTCR_DINCOS_MASK); + ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) | + STM32_MDMA_CTCR_DINCOS(dst_bus_width); + + /* Set memory burst value */ + dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width; + dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, + dst_maxburst, + dst_addr_width); + ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK; + ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst))); + + /* Select bus */ + stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, + src_addr); + + if (dst_bus_width != src_bus_width) + ctcr |= STM32_MDMA_CTCR_PKE; + + /* Set source address */ + stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr); + break; + + default: + dev_err(chan2dev(chan), "Dma direction is not supported\n"); + return -EINVAL; + } + + *mdma_ccr = ccr; + *mdma_ctcr = ctcr; + *mdma_ctbr = ctbr; + + return 0; +} + +static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan, + struct stm32_mdma_hwdesc *hwdesc) +{ + dev_dbg(chan2dev(chan), "hwdesc: 0x%p\n", hwdesc); + dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", hwdesc->ctcr); + dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", hwdesc->cbndtr); + dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", hwdesc->csar); + dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", hwdesc->cdar); + dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", hwdesc->cbrur); + dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", hwdesc->clar); + dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", hwdesc->ctbr); + dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", hwdesc->cmar); + dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", hwdesc->cmdr); +} + +static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan, + struct stm32_mdma_desc *desc, + enum dma_transfer_direction dir, u32 count, + dma_addr_t src_addr, dma_addr_t dst_addr, + u32 len, u32 ctcr, u32 ctbr, bool is_last, + bool is_first, bool |