// SPDX-License-Identifier: GPL-2.0
//
// Copyright (C) 2018 Socionext Inc.
// Author: Masahiro Yamada <yamada.masahiro@socionext.com>
#include <linux/bits.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/types.h>
#include "virt-dma.h"
/* registers common for all channels */
#define UNIPHIER_MDMAC_CMD 0x000 /* issue DMA start/abort */
#define UNIPHIER_MDMAC_CMD_ABORT BIT(31) /* 1: abort, 0: start */
/* per-channel registers */
#define UNIPHIER_MDMAC_CH_OFFSET 0x100
#define UNIPHIER_MDMAC_CH_STRIDE 0x040
#define UNIPHIER_MDMAC_CH_IRQ_STAT 0x010 /* current hw status (RO) */
#define UNIPHIER_MDMAC_CH_IRQ_REQ 0x014 /* latched STAT (WOC) */
#define UNIPHIER_MDMAC_CH_IRQ_EN 0x018 /* IRQ enable mask */
#define UNIPHIER_MDMAC_CH_IRQ_DET 0x01c /* REQ & EN (RO) */
#define UNIPHIER_MDMAC_CH_IRQ__ABORT BIT(13)
#define UNIPHIER_MDMAC_CH_IRQ__DONE BIT(1)
#define UNIPHIER_MDMAC_CH_SRC_MODE 0x020 /* mode of source */
#define UNIPHIER_MDMAC_CH_DEST_MODE 0x024 /* mode of destination */
#define UNIPHIER_MDMAC_CH_MODE__ADDR_INC (0 << 4)
#define UNIPHIER_MDMAC_CH_MODE__ADDR_DEC (1 << 4)
#define UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED (2 << 4)
#define UNIPHIER_MDMAC_CH_SRC_ADDR 0x028 /* source address */
#define UNIPHIER_MDMAC_CH_DEST_ADDR 0x02c /* destination address */
#define UNIPHIER_MDMAC_CH_SIZE 0x030 /* transfer bytes */
#define UNIPHIER_MDMAC_SLAVE_BUSWIDTHS \
(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
struct uniphier_mdmac_desc {
struct virt_dma_desc vd;
struct scatterlist *sgl;
unsigned int sg_len;
unsigned int sg_cur;
enum dma_transfer_direction dir;
};
struct uniphier_mdmac_chan {
struct virt_dma_chan vc;
struct uniphier_mdmac_device *mdev;
struct uniphier_mdmac_desc *md;
void __iomem *reg_ch_base;
unsigned int chan_id;
};
struct uniphier_mdmac_device {
struct dma_device ddev;
struct clk *clk;
void __iomem *reg_base;
struct uniphier_mdmac_chan channels[];
};
static struct uniphier_mdmac_chan *
to_uniphier_mdmac_chan(struct virt_dma_chan *vc)
{
return container_of(vc, struct uniphier_mdmac_chan, vc);
}
static struct uniphier_mdmac_desc *
to_uniphier_mdmac_desc(struct virt_dma_desc *vd)
{
return container_of(vd, struct uniphier_mdmac_desc, vd);
}
/* mc->vc.lock must be held by caller */
static struct uniphier_mdmac_desc *
uniphier_mdmac_next_desc(struct uniphier_mdmac_chan *mc)
{
struct virt_dma_desc *vd;
vd = vchan_next_desc(&mc->vc);
if (!vd) {
mc->md = NULL;
return NULL;
}
list_del(&vd->node);
mc->md = to_uniphier_mdmac_desc(vd);
return mc->md;
}
/* mc->vc.lock must be held by caller */
static void uniphier_mdmac_handle(struct uniphier_mdmac_chan *mc,
struct uniphier_mdmac_desc *md)
{
struct uniphier_mdmac_device *mdev = mc->mdev;
struct scatterlist *sg;
u32 irq_flag = UNIPHIER_MDMAC_CH_IRQ__DONE;
u32 src_mode, src_addr, dest_mode, dest_addr, chunk_size;
sg = &md->sgl[md->sg_cur];
if (md->dir == DMA_MEM_TO_DEV) {
src_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_INC;
src_addr = sg_dma_address(sg);
dest_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED;
dest_addr = 0;
} else {
src_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED;
src_addr = 0;
dest_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_INC;
dest_addr = sg_dma_address(sg);
}
chunk_size = sg_dma_len(sg);
writel(src_mode, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SRC_MODE);
writel(dest_mode, mc->reg_ch_base + UNIPHIER_MDMAC_CH_DEST_MODE);
writel(src_addr, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SRC_ADDR);
writel(dest_addr, mc->reg_ch_base + UNIPHIER_MDMAC_CH_DEST_ADDR);
writel(chunk_size, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SIZE);
/* write 1 to clear */
writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ);
writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_EN);
writel(BIT(mc->chan_id), mdev->reg_base + UNIPHIER_MDMAC_CMD);
}
/* mc->vc.lock must be held by caller */
static void uniphier_mdmac_start(struct uniphier_mdmac_chan *mc)
{
struct uniphier_mdmac_desc *md;
md = uniphier_mdmac_next_desc(mc);
if (md)
uniphier_mdmac_handle(mc, md);
}
/* mc->vc.lock must be held by caller */
static int uniphier_mdmac_abort(struct uniphier_mdmac_chan *mc)
{
struct uniphier_mdmac_device *mdev = mc->mdev;
u32 irq_flag = UNIPHIER_MDMAC_CH_IRQ__ABORT;
u32 val;
/* write 1 to clear */
writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ);
writel(UNIPHIER_MDMAC_CMD_ABORT | BIT(mc->chan_id),
mdev->reg_base + UNIPHIER_MDMAC_CMD);
/*
* Abort should be accepted soon. We poll the bit here instead of
* waiting for the interrupt.
*/
return readl_poll_timeout(mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ,
val, val & irq_flag, 0, 20);
}
static irqreturn_t uniphier_mdmac_interrupt(int irq, void *dev_id)
{
struct uniphier_mdmac_chan *mc = dev_id;
struct uniphier_mdmac_desc *md;
irqreturn_t ret = IRQ_HANDLED;
u32 irq_stat;
spin_lock(&mc->vc.lock);
irq_stat = readl(mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_DET);
/*
* Some channels share a single interrupt line. If the IRQ status is 0,
* this is probably triggered by a different channel.
*/
if (!irq_stat) {
ret = IRQ_NONE;
goto out;
}
/* write 1 to clear */
writel(irq_stat, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ);
/*
* UNIPHIER_MDMAC_CH_IRQ__DONE interrupt is asserted even when the DMA
* is aborted. To distinguish the normal completion and the abort,
* check mc->md. If it is NULL, we are aborting.
*/
md = mc->md;
if (!md)
goto out;
md->sg_cur++;
if (md->sg_cur >= md->sg_len) {
vchan_cookie_complete(&md->vd);
md = uniphier_mdmac_next_desc(mc);
if (!md)
goto out;
}
uniphier_mdmac_handle(mc, md);
out:
spin_unlock(&mc->vc.lock);
return ret;
}
static void uniphier_mdmac_free_chan_resources(struct dma_chan *chan)
{
vchan_free_chan_resources(to_virt_chan(chan));
}
static struct dma_async_tx_descriptor *
uniphier_mdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len,
enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct virt_dma_chan *vc = to_virt_chan(chan);
struct uniphier_mdmac_desc *md;
if (!is_slave_direction(direction))
return NULL;
md = kzalloc(sizeof(*md), GFP_NOWAIT);
if (!md)
return NULL;
md->sgl = sgl;
md->sg_len = sg_len;
md->dir = direction;
return vchan_tx_prep(vc, &md->vd, flags);
}
static int uniphier_mdmac_terminate_all(struct dma_chan *chan)
{
struct virt_dma_chan *vc = to_virt_chan(chan);
struct uniphier_mdmac_chan *mc = to_uniphier_mdmac_chan(vc);
unsigned long flags;
int ret = 0;
LIST_HEAD(head);
spin_lock_irqsave(&vc->lock, flags);
if (mc->md) {
vchan_terminate_vdesc(&mc->md->vd);
mc->md = NULL;
ret = uniphier_mdmac_abort(mc);
}
vchan_get_all_descriptors(vc, &head);
spin_unlock_irqrestore(&vc->lock, flags);
vchan_dma_desc_free_list(vc, &head);
return ret;
}
static void uniphier_mdmac_synchronize(struct dma_chan *chan)
{
vchan_synchronize(to_virt_chan(chan));
}
static