// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2015 Linaro.
*/
#include <linux/sched.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/of_device.h>
#include <linux/of.h>
#include <linux/clk.h>
#include <linux/of_dma.h>
#include "virt-dma.h"
#define DRIVER_NAME "zx-dma"
#define DMA_ALIGN 4
#define DMA_MAX_SIZE (0x10000 - 512)
#define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
#define REG_ZX_SRC_ADDR 0x00
#define REG_ZX_DST_ADDR 0x04
#define REG_ZX_TX_X_COUNT 0x08
#define REG_ZX_TX_ZY_COUNT 0x0c
#define REG_ZX_SRC_ZY_STEP 0x10
#define REG_ZX_DST_ZY_STEP 0x14
#define REG_ZX_LLI_ADDR 0x1c
#define REG_ZX_CTRL 0x20
#define REG_ZX_TC_IRQ 0x800
#define REG_ZX_SRC_ERR_IRQ 0x804
#define REG_ZX_DST_ERR_IRQ 0x808
#define REG_ZX_CFG_ERR_IRQ 0x80c
#define REG_ZX_TC_IRQ_RAW 0x810
#define REG_ZX_SRC_ERR_IRQ_RAW 0x814
#define REG_ZX_DST_ERR_IRQ_RAW 0x818
#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
#define REG_ZX_STATUS 0x820
#define REG_ZX_DMA_GRP_PRIO 0x824
#define REG_ZX_DMA_ARB 0x828
#define ZX_FORCE_CLOSE BIT(31)
#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
#define ZX_MAX_BURST_LEN 16
#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
#define ZX_IRQ_ENABLE_ALL (3 << 4)
#define ZX_DST_FIFO_MODE BIT(3)
#define ZX_SRC_FIFO_MODE BIT(2)
#define ZX_SOFT_REQ BIT(1)
#define ZX_CH_ENABLE BIT(0)
#define ZX_DMA_BUSWIDTHS \
(BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
enum zx_dma_burst_width {
ZX_DMA_WIDTH_8BIT = 0,
ZX_DMA_WIDTH_16BIT = 1,
ZX_DMA_WIDTH_32BIT = 2,
ZX_DMA_WIDTH_64BIT = 3,
};
struct zx_desc_hw {
u32 saddr;
u32 daddr;
u32 src_x;
u32 src_zy;
u32 src_zy_step;
u32 dst_zy_step;
u32 reserved1;
u32 lli;
u32 ctr;
u32 reserved[7]; /* pack as hardware registers region size */
} __aligned(32);
struct zx_dma_desc_sw {
struct virt_dma_desc vd;
dma_addr_t desc_hw_lli;
size_t desc_num;
size_t size;
struct zx_desc_hw *desc_hw;
};
struct zx_dma_phy;
struct zx_dma_chan {
struct dma_slave_config slave_cfg;
int id; /* Request phy chan id */
u32 ccfg;
u32 cyclic;
struct virt_dma_chan vc;
struct zx_dma_phy *phy;
struct list_head node;
dma_addr_t dev_addr;
enum dma_status status;
};
struct zx_dma_phy {
u32 idx;
void __iomem *base;
struct zx_dma_chan *vchan;
struct zx_dma_desc_sw *ds_run;
struct zx_dma_desc_sw *ds_done;
};
struct zx_dma_dev {
struct dma_device slave;
void __iomem *base;
spinlock_t lock; /* lock for ch and phy */
struct list_head chan_pending;
struct zx_dma_phy *phy;
struct zx_dma_chan *chans;
struct clk *clk;
struct dma_pool *pool;
u32 dma_channels;
u32 dma_requests;
int irq;
};
#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan)
{
return container_of(chan, struct zx_dma_chan, vc.chan);
}
static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d)
{
u32 val = 0;
val = readl_relaxed(phy->base + REG_ZX_CTRL);
val &= ~ZX_CH_ENABLE;
val |= ZX_FORCE_CLOSE;
writel_relaxed(val, phy->base + REG_ZX_CTRL);
val = 0x1 << phy->idx;
writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW);
writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW);
writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
}
static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw)
{
writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR);
writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR);
writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT);
writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT);
writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP);
writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP);
writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR);
writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL);
}
static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy)
{
return readl_relaxed(phy->base + REG_ZX_LLI_ADDR);
}
static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d)
{
return readl_relaxed(d->base + REG_ZX_STATUS);
}
static void zx_dma_init_state(struct zx_dma_dev *d)
{
/* set same priority */
writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB);
/* clear all irq */
writel_relaxed(0xffffffff, d->base