From 47ec7f09bc107720905c96bc37771e4ed1ff0aed Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 13 May 2020 11:47:49 -0700 Subject: dmaengine: cookie bypass for out of order completion The cookie tracking in dmaengine expects all submissions completed in order. Some DMA devices like Intel DSA can complete submissions out of order, especially if configured with a work queue sharing multiple DMA engines. Add a status DMA_OUT_OF_ORDER that tx_status can be returned for those DMA devices. The user should use callbacks to track the completion rather than the DMA cookie. This would address the issue of dmatest complaining that descriptors are "busy" when the cookie count goes backwards due to out of order completion. Add DMA_COMPLETION_NO_ORDER DMA capability to allow the driver to flag the device's ability to complete operations out of order. Reported-by: Swathi Kovvuri Signed-off-by: Dave Jiang Tested-by: Swathi Kovvuri Link: https://lore.kernel.org/r/158939557151.20335.12404113976045569870.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index e1c03339918f..9f9a13a2c01f 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -39,6 +39,7 @@ enum dma_status { DMA_IN_PROGRESS, DMA_PAUSED, DMA_ERROR, + DMA_OUT_OF_ORDER, }; /** @@ -61,6 +62,7 @@ enum dma_transaction_type { DMA_SLAVE, DMA_CYCLIC, DMA_INTERLEAVE, + DMA_COMPLETION_NO_ORDER, /* last transaction type for creation of the capabilities mask */ DMA_TX_TYPE_END, }; -- cgit v1.2.3 From 2be90e914c12bdea70b0bd297d5715ee66eb46d0 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 17 Jul 2020 19:51:18 -0700 Subject: dmaengine: linux/dmaengine.h: drop duplicated word in a comment Drop the doubled word "has" in a comment. Signed-off-by: Randy Dunlap Cc: Vinod Koul Cc: dmaengine@vger.kernel.org Link: https://lore.kernel.org/r/06e64046-ebf1-15db-dbaf-73698de3b493@infradead.org Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 9f9a13a2c01f..883e1e087de5 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -164,7 +164,7 @@ struct dma_interleaved_template { * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of * this transaction * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client - * acknowledges receipt, i.e. has has a chance to establish any dependency + * acknowledges receipt, i.e. has a chance to establish any dependency * chains * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P -- cgit v1.2.3 From d97758e048e5fe91c7d8ff9ce5f030ee88d92161 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 23 Jul 2020 03:58:41 +0300 Subject: dmaengine: Introduce min burst length capability Some hardware aside from default 0/1 may have greater minimum burst transactions length constraints. Here we introduce the DMA device and slave capability, which if required can be initialized by the DMA engine driver with the device-specific value. Signed-off-by: Serge Semin Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200723005848.31907-4-Sergey.Semin@baikalelectronics.ru Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 883e1e087de5..7d6e2aa26980 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -467,6 +467,7 @@ enum dma_residue_granularity { * Since the enum dma_transfer_direction is not defined as bit flag for * each type, the dma controller should set BIT() and same * should be checked by controller as well + * @min_burst: min burst capability per-transfer * @max_burst: max burst capability per-transfer * @cmd_pause: true, if pause is supported (i.e. for reading residue or * for resume later) @@ -480,6 +481,7 @@ struct dma_slave_caps { u32 src_addr_widths; u32 dst_addr_widths; u32 directions; + u32 min_burst; u32 max_burst; bool cmd_pause; bool cmd_resume; @@ -771,6 +773,7 @@ struct dma_filter { * Since the enum dma_transfer_direction is not defined as bit flag for * each type, the dma controller should set BIT() and same * should be checked by controller as well + * @min_burst: min burst capability per-transfer * @max_burst: max burst capability per-transfer * @residue_granularity: granularity of the transfer residue reported * by tx_status @@ -841,6 +844,7 @@ struct dma_device { u32 src_addr_widths; u32 dst_addr_widths; u32 directions; + u32 min_burst; u32 max_burst; bool descriptor_reuse; enum dma_residue_granularity residue_granularity; -- cgit v1.2.3 From b1b40b8fe7e8fb26e33bad1766ce322d2c63a6c7 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 23 Jul 2020 03:58:42 +0300 Subject: dmaengine: Introduce max SG burst capability Some devices may lack the support of the hardware accelerated SG list entries automatic walking through and execution. In this case a burden of the SG list traversal and DMA engine re-initialization lies on the DMA engine driver (normally implemented by using a DMA transfer completion IRQ to recharge the DMA device with a next SG list entry). But such solution may not be suitable for some DMA consumers. In particular SPI devices need both Tx and Rx DMA channels work synchronously in order to avoid the Rx FIFO overflow. In case if Rx DMA channel is paused for some time while the Tx DMA channel works implicitly pulling data into the Rx FIFO, the later will be eventually overflown, which will cause the data loss. So if SG list entries aren't automatically fetched by the DMA engine, but are one-by-one manually selected for execution in the ISRs/deferred work/etc., such problem will eventually happen due to the non-deterministic latencies of the service execution. In order to let the DMA consumer know about the DMA device capabilities regarding the hardware accelerated SG list traversal we introduce the max_sg_burst capability. It is supposed to be initialized by the DMA engine driver with 0 if there is no limitation of the number of SG entries atomically executed and with non-zero value if there is such constraints, so the upper limit is determined by the number set to the property. Suggested-by: Andy Shevchenko Signed-off-by: Serge Semin Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200723005848.31907-5-Sergey.Semin@baikalelectronics.ru Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 7d6e2aa26980..4cbe09e66db2 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -469,6 +469,9 @@ enum dma_residue_granularity { * should be checked by controller as well * @min_burst: min burst capability per-transfer * @max_burst: max burst capability per-transfer + * @max_sg_burst: max number of SG list entries executed in a single burst + * DMA tansaction with no software intervention for reinitialization. + * Zero value means unlimited number of entries. * @cmd_pause: true, if pause is supported (i.e. for reading residue or * for resume later) * @cmd_resume: true, if resume is supported @@ -483,6 +486,7 @@ struct dma_slave_caps { u32 directions; u32 min_burst; u32 max_burst; + u32 max_sg_burst; bool cmd_pause; bool cmd_resume; bool cmd_terminate; @@ -775,6 +779,9 @@ struct dma_filter { * should be checked by controller as well * @min_burst: min burst capability per-transfer * @max_burst: max burst capability per-transfer + * @max_sg_burst: max number of SG list entries executed in a single burst + * DMA tansaction with no software intervention for reinitialization. + * Zero value means unlimited number of entries. * @residue_granularity: granularity of the transfer residue reported * by tx_status * @device_alloc_chan_resources: allocate resources and return the @@ -846,6 +853,7 @@ struct dma_device { u32 directions; u32 min_burst; u32 max_burst; + u32 max_sg_burst; bool descriptor_reuse; enum dma_residue_granularity residue_granularity; -- cgit v1.2.3 From 3b6d694eb3eebd86ec44a119e730943ac8e03a6b Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 23 Jul 2020 03:58:43 +0300 Subject: dmaengine: Introduce DMA-device device_caps callback There are DMA devices (like ours version of Synopsys DW DMAC) which have DMA capabilities non-uniformly redistributed between the device channels. In order to provide a way of exposing the channel-specific parameters to the DMA engine consumers, we introduce a new DMA-device callback. In case if provided it gets called from the dma_get_slave_caps() method and is able to override the generic DMA-device capabilities. Signed-off-by: Serge Semin Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200723005848.31907-6-Sergey.Semin@baikalelectronics.ru Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 4cbe09e66db2..d718671bfd25 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -801,6 +801,8 @@ struct dma_filter { * be called after period_len bytes have been transferred. * @device_prep_interleaved_dma: Transfer expression in a generic way. * @device_prep_dma_imm_data: DMA's 8 byte immediate data to the dst address + * @device_caps: May be used to override the generic DMA slave capabilities + * with per-channel specific ones * @device_config: Pushes a new configuration to a channel, return 0 or an error * code * @device_pause: Pauses any transfer happening on a channel. Returns @@ -901,6 +903,8 @@ struct dma_device { struct dma_chan *chan, dma_addr_t dst, u64 data, unsigned long flags); + void (*device_caps)(struct dma_chan *chan, + struct dma_slave_caps *caps); int (*device_config)(struct dma_chan *chan, struct dma_slave_config *config); int (*device_pause)(struct dma_chan *chan); -- cgit v1.2.3 From 585d35451e94b2e1b0bf59ef55d3b4a1c8ab3d77 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 23 Jul 2020 03:58:46 +0300 Subject: dmaengine: dw: Initialize min and max burst DMA device capability According to the DW APB DMAC data book the minimum burst transaction length is 1 and it's true for any version of the controller since isn't parametrised in the coreAssembler so can't be changed at the IP-core synthesis stage. The maximum burst transaction can vary from channel to channel and from controller to controller depending on a IP-core parameter the system engineer activated during the IP-core synthesis. Let's initialise both min_burst and max_burst members of the DMA controller descriptor with extreme values so the DMA clients could use them to properly optimize the DMA requests. The channels and controller-specific max_burst length initialization will be introduced by the follow-up patches. Signed-off-by: Serge Semin Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200723005848.31907-9-Sergey.Semin@baikalelectronics.ru Signed-off-by: Vinod Koul --- include/linux/platform_data/dma-dw.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h index f3eaf9ec00a1..369e41e9dcc9 100644 --- a/include/linux/platform_data/dma-dw.h +++ b/include/linux/platform_data/dma-dw.h @@ -12,6 +12,8 @@ #define DW_DMA_MAX_NR_MASTERS 4 #define DW_DMA_MAX_NR_CHANNELS 8 +#define DW_DMA_MIN_BURST 1 +#define DW_DMA_MAX_BURST 256 /** * struct dw_dma_slave - Controller-specific information about a slave -- cgit v1.2.3 From ca7f2851712e7072e8f327882dc4bdaaae3a8079 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 23 Jul 2020 03:58:47 +0300 Subject: dmaengine: dw: Introduce max burst length hw config IP core of the DW DMA controller may be synthesized with different max burst length of the transfers per each channel. According to Synopsis having the fixed maximum burst transactions length may provide some performance gain. At the same time setting up the source and destination multi size exceeding the max burst length limitation may cause a serious problems. In our case the DMA transaction just hangs up. In order to fix this lets introduce the max burst length platform config of the DW DMA controller device and don't let the DMA channels configuration code exceed the burst length hardware limitation. Note the maximum burst length parameter can be detected either in runtime from the DWC parameter registers or from the dedicated DT property. Depending on the IP core configuration the maximum value can vary from channel to channel so by overriding the channel slave max_burst capability we make sure a DMA consumer will get the channel-specific max burst length. Signed-off-by: Serge Semin Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200723005848.31907-10-Sergey.Semin@baikalelectronics.ru Signed-off-by: Vinod Koul --- include/linux/platform_data/dma-dw.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h index 369e41e9dcc9..4f681df85c27 100644 --- a/include/linux/platform_data/dma-dw.h +++ b/include/linux/platform_data/dma-dw.h @@ -44,6 +44,8 @@ struct dw_dma_slave { * @data_width: Maximum data width supported by hardware per AHB master * (in bytes, power of 2) * @multi_block: Multi block transfers supported by hardware per channel. + * @max_burst: Maximum value of burst transaction size supported by hardware + * per channel (in units of CTL.SRC_TR_WIDTH/CTL.DST_TR_WIDTH). * @protctl: Protection control signals setting per channel. */ struct dw_dma_platform_data { @@ -58,6 +60,7 @@ struct dw_dma_platform_data { unsigned char nr_masters; unsigned char data_width[DW_DMA_MAX_NR_MASTERS]; unsigned char multi_block[DW_DMA_MAX_NR_CHANNELS]; + u32 max_burst[DW_DMA_MAX_NR_CHANNELS]; #define CHAN_PROTCTL_PRIVILEGED BIT(0) #define CHAN_PROTCTL_BUFFERABLE BIT(1) #define CHAN_PROTCTL_CACHEABLE BIT(2) -- cgit v1.2.3 From 1a0c02ba643ed05c07ddf14d87c3bec640666836 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Mon, 20 Jul 2020 08:50:58 -0700 Subject: dmaengine: idxd: add missing invalid flags field to completion Add missing "invalid flags" field to completion record struct. Reported-by: Nikhil Rao Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/159526025819.49266.13176787210106133664.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- include/uapi/linux/idxd.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include') diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index 1f412fbf561b..5d07a3ff9e96 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -178,6 +178,12 @@ struct dsa_completion_record { uint32_t bytes_completed; uint64_t fault_addr; union { + /* common record */ + struct { + uint32_t invalid_flags:24; + uint32_t rsvd2:8; + }; + uint16_t delta_rec_size; uint16_t crc_val; -- cgit v1.2.3 From 6bd0dffa1a6e19e73964ae47c964f57c625cce05 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 21 Jul 2020 16:08:44 +0300 Subject: dmaengine: dw: Don't include unneeded header to platform data header Including device.h is too much for the dma-dw.h platform data header. Replace it with the headers of which dma-dw.h is direct user. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200721130844.64162-1-andriy.shevchenko@linux.intel.com Signed-off-by: Vinod Koul --- include/linux/platform_data/dma-dw.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h index 4f681df85c27..fbbeb2f6189b 100644 --- a/include/linux/platform_data/dma-dw.h +++ b/include/linux/platform_data/dma-dw.h @@ -8,13 +8,16 @@ #ifndef _PLATFORM_DATA_DMA_DW_H #define _PLATFORM_DATA_DMA_DW_H -#include +#include +#include #define DW_DMA_MAX_NR_MASTERS 4 #define DW_DMA_MAX_NR_CHANNELS 8 #define DW_DMA_MIN_BURST 1 #define DW_DMA_MAX_BURST 256 +struct device; + /** * struct dw_dma_slave - Controller-specific information about a slave * -- cgit v1.2.3