// SPDX-License-Identifier: GPL-2.0-only
/*
* Intel I/OAT DMA Linux driver
* Copyright(c) 2004 - 2015 Intel Corporation.
*/
/*
* This driver supports an Intel I/OAT DMA engine, which does asynchronous
* copy operations.
*/
#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/prefetch.h>
#include <linux/sizes.h>
#include "dma.h"
#include "registers.h"
#include "hw.h"
#include "../dmaengine.h"
int completion_timeout = 200;
module_param(completion_timeout, int, 0644);
MODULE_PARM_DESC(completion_timeout,
"set ioat completion timeout [msec] (default 200 [msec])");
int idle_timeout = 2000;
module_param(idle_timeout, int, 0644);
MODULE_PARM_DESC(idle_timeout,
"set ioat idel timeout [msec] (default 2000 [msec])");
#define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout)
#define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)
static char *chanerr_str[] = {
"DMA Transfer Source Address Error",
"DMA Transfer Destination Address Error",
"Next Descriptor Address Error",
"Descriptor Error",
"Chan Address Value Error",
"CHANCMD Error",
"Chipset Uncorrectable Data Integrity Error",
"DMA Uncorrectable Data Integrity Error",
"Read Data Error",
"Write Data Error",
"Descriptor Control Error",
"Descriptor Transfer Size Error",
"Completion Address Error",
"Interrupt Configuration Error",
"Super extended descriptor Address Error",
"Unaffiliated Error",
"CRC or XOR P Error",
"XOR Q Error",
"Descriptor Count Error",
"DIF All F detect Error",
"Guard Tag verification Error",
"Application Tag verification Error",
"Reference Tag verification Error",
"Bundle Bit Error",
"Result DIF All F detect Error",
"Result Guard Tag verification Error",
"Result Application Tag verification Error",
"Result Reference Tag verification Error",
};
static void ioat_eh(struct ioatdma_chan *ioat_chan);
static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
{
int i;
for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
if ((chanerr >> i) & 1) {
dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
i, chanerr_str[i]);
}
}
}
/**
* ioat_dma_do_interrupt - handler used for single vector interrupt mode
* @irq: interrupt id
* @data: interrupt data
*/
irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{
struct ioatdma_device *instance = data;
struct ioatdma_chan *ioat_chan;
unsigned long attnstatus;
int bit;
u8 intrctrl;
intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
return IRQ_NONE;
if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
return IRQ_NONE;
}
attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
ioat_chan = ioat_chan_by_index(instance, bit);
if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&ioat_chan->cleanup_task);
}
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
return IRQ_HANDLED;
}
/**
* ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
* @irq: interrupt id
* @data: interrupt data
*/
irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{
struct ioatdma_chan *ioat_chan = data;
if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&ioat_chan->cleanup_task);
return IRQ_HANDLED;
}
void ioat_stop(struct ioatdma_chan *ioat_chan)
{
struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
struct pci_dev *pdev = ioat_dma->pdev;
int chan_id = chan_num(ioat_chan);
struct msix_entry *msix;
/* 1/ stop irq from firing tasklets
* 2/ stop the tasklet from re-arming irqs
*/
clear_bit(IOAT_RUN, &ioat_chan->state);
/* flush inflight interrupts */
switch (ioat_dma->irq_mode) {
case IOAT_MSIX:
msix = &ioat_dma->msix_entries[chan_id];
synchronize_irq(msix->vector);
break;
case IOAT_MSI:
case IOAT_IN