1 files changed, 1219 insertions, 601 deletions

diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 6f6c602d04af..8de72f3fba4d 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -40,859 +42,1475 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- *   PCIe NTB Perf Linux driver
+ * PCIe NTB Perf Linux driver
+ */
+
+/*
+ * How to use this tool, by example.
+ *
+ * Assuming $DBG_DIR is something like:
+ * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
+ * Suppose aside from local device there is at least one remote device
+ * connected to NTB with index 0.
+ *-----------------------------------------------------------------------------
+ * Eg: install driver with specified chunk/total orders and dma-enabled flag
+ *
+ * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
+ *-----------------------------------------------------------------------------
+ * Eg: check NTB ports (index) and MW mapping information
+ *
+ * root@self# cat $DBG_DIR/info
+ *-----------------------------------------------------------------------------
+ * Eg: start performance test with peer (index 0) and get the test metrics
+ *
+ * root@self# echo 0 > $DBG_DIR/run
+ * root@self# cat $DBG_DIR/run
  */
 
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/kthread.h>
-#include <linux/time.h>
-#include <linux/timer.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/pci.h>
+#include <linux/ktime.h>
 #include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/debugfs.h>
-#include <linux/dmaengine.h>
 #include <linux/delay.h>
 #include <linux/sizes.h>
+#include <linux/workqueue.h>
+#include <linux/debugfs.h>
+#include <linux/random.h>
 #include <linux/ntb.h>
-#include <linux/mutex.h>
 
 #define DRIVER_NAME		"ntb_perf"
-#define DRIVER_DESCRIPTION	"PCIe NTB Performance Measurement Tool"
-
-#define DRIVER_VERSION		"1.0"
-#define DRIVER_AUTHOR		"Dave Jiang <dave.jiang@intel.com>"
-
-#define PERF_LINK_DOWN_TIMEOUT	10
-#define PERF_VERSION		0xffff0001
-#define MAX_THREADS		32
-#define MAX_TEST_SIZE		SZ_1M
-#define MAX_SRCS		32
-#define DMA_OUT_RESOURCE_TO	msecs_to_jiffies(50)
-#define DMA_RETRIES		20
-#define SZ_4G			(1ULL << 32)
-#define MAX_SEG_ORDER		20 /* no larger than 1M for kmalloc buffer */
-#define PIDX			NTB_DEF_PEER_IDX
+#define DRIVER_VERSION		"2.0"
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRIVER_VERSION);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
+MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
+
+#define MAX_THREADS_CNT		32
+#define DEF_THREADS_CNT		1
+#define MAX_CHUNK_SIZE		SZ_1M
+#define MAX_CHUNK_ORDER		20 /* no larger than 1M */
+
+#define DMA_TRIES		100
+#define DMA_MDELAY		10
 
-static struct dentry *perf_debugfs_dir;
+#define MSG_TRIES		500
+#define MSG_UDELAY_LOW		1000
+#define MSG_UDELAY_HIGH		2000
+
+#define PERF_BUF_LEN 1024
 
 static unsigned long max_mw_size;
 module_param(max_mw_size, ulong, 0644);
-MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
+MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
 
-static unsigned int seg_order = 19; /* 512K */
-module_param(seg_order, uint, 0644);
-MODULE_PARM_DESC(seg_order, "size order [2^n] of buffer segment for testing");
+static unsigned char chunk_order = 19; /* 512K */
+module_param(chunk_order, byte, 0644);
+MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
 
-static unsigned int run_order = 32; /* 4G */
-module_param(run_order, uint, 0644);
-MODULE_PARM_DESC(run_order, "size order [2^n] of total data to transfer");
+static unsigned char total_order = 30; /* 1G */
+module_param(total_order, byte, 0644);
+MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
 
 static bool use_dma; /* default to 0 */
 module_param(use_dma, bool, 0644);
-MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
-
-static bool on_node = true; /* default to 1 */
-module_param(on_node, bool, 0644);
-MODULE_PARM_DESC(on_node, "Run threads only on NTB device node (default: true)");
-
-struct perf_mw {
-	phys_addr_t	phys_addr;
-	resource_size_t	phys_size;
-	void __iomem	*vbase;
-	size_t		xlat_size;
-	size_t		buf_size;
-	void		*virt_addr;
-	dma_addr_t	dma_addr;
+MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
+
+/*==============================================================================
+ *                         Perf driver data definition
+ *==============================================================================
+ */
+
+enum perf_cmd {
+	PERF_CMD_INVAL = -1,/* invalid spad command */
+	PERF_CMD_SSIZE = 0, /* send out buffer size */
+	PERF_CMD_RSIZE = 1, /* recv in  buffer size */
+	PERF_CMD_SXLAT = 2, /* send in  buffer xlat */
+	PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
+	PERF_CMD_CLEAR = 4, /* clear allocated memory */
+	PERF_STS_DONE  = 5, /* init is done */
+	PERF_STS_LNKUP = 6, /* link up state flag */
 };
 
 struct perf_ctx;
 
-struct pthr_ctx {
-	struct task_struct	*thread;
-	struct perf_ctx		*perf;
-	atomic_t		dma_sync;
-	struct dma_chan		*dma_chan;
-	int			dma_prep_err;
-	int			src_idx;
-	void			*srcs[MAX_SRCS];
-	wait_queue_head_t       *wq;
-	int			status;
-	u64			copied;
-	u64			diff_us;
+struct perf_peer {
+	struct perf_ctx	*perf;
+	int pidx;
+	int gidx;
+
+	/* Outbound MW params */
+	u64 outbuf_xlat;
+	resource_size_t outbuf_size;
+	void __iomem *outbuf;
+
+	/* Inbound MW params */
+	dma_addr_t inbuf_xlat;
+	resource_size_t inbuf_size;
+	void		*inbuf;
+
+	/* NTB connection setup service */
+	struct work_struct	service;
+	unsigned long		sts;
 };
+#define to_peer_service(__work) \
+	container_of(__work, struct perf_peer, service)
 
-struct perf_ctx {
-	struct ntb_dev		*ntb;
-	spinlock_t		db_lock;
-	struct perf_mw		mw;
-	bool			link_is_up;
-	struct delayed_work	link_work;
-	wait_queue_head_t	link_wq;
-	u8			perf_threads;
-	/* mutex ensures only one set of threads run at once */
-	struct mutex		run_mutex;
-	struct pthr_ctx		pthr_ctx[MAX_THREADS];
-	atomic_t		tsync;
-	atomic_t                tdone;
+struct perf_thread {
+	struct perf_ctx *perf;
+	int tidx;
+
+	/* DMA-based test sync parameters */
+	atomic_t dma_sync;
+	wait_queue_head_t dma_wait;
+	struct dma_chan *dma_chan;
+
+	/* Data source and measured statistics */
+	void *src;
+	u64 copied;
+	ktime_t duration;
+	int status;
+	struct work_struct work;
 };
+#define to_thread_work(__work) \
+	container_of(__work, struct perf_thread, work)
 
-enum {
-	VERSION = 0,
-	MW_SZ_HIGH,
-	MW_SZ_LOW,
-	MAX_SPAD
+struct perf_ctx {
+	struct ntb_dev *ntb;
+
+	/* Global device index and peers descriptors */
+	int gidx;
+	int pcnt;
+	struct perf_peer *peers;
+
+	/* Performance measuring work-threads interface */
+	unsigned long busy_flag;
+	wait_queue_head_t twait;
+	atomic_t tsync;
+	u8 tcnt;
+	struct perf_peer *test_peer;
+	struct perf_thread threads[MAX_THREADS_CNT];
+
+	/* Scratchpad/Message IO operations */
+	int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
+	int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
+			u64 *data);
+
+	struct dentry *dbgfs_dir;
 };
 
+/*
+ * Scratchpads-base commands interface
+ */
+#define PERF_SPAD_CNT(_pcnt) \
+	(3*((_pcnt) + 1))
+#define PERF_SPAD_CMD(_gidx) \
+	(3*(_gidx))
+#define PERF_SPAD_LDATA(_gidx) \
+	(3*(_gidx) + 1)
+#define PERF_SPAD_HDATA(_gidx) \
+	(3*(_gidx) + 2)
+#define PERF_SPAD_NOTIFY(_gidx) \
+	(BIT_ULL(_gidx))
+
+/*
+ * Messages-base commands interface
+ */
+#define PERF_MSG_CNT		3
+#define PERF_MSG_CMD		0
+#define PERF_MSG_LDATA		1
+#define PERF_MSG_HDATA		2
+
+/*==============================================================================
+ *                           Static data declarations
+ *==============================================================================
+ */
+
+static struct dentry *perf_dbgfs_topdir;
+
+static struct workqueue_struct *perf_wq __read_mostly;
+
+/*==============================================================================
+ *                  NTB cross-link commands execution service
+ *==============================================================================
+ */
+
+static void perf_terminate_test(struct perf_ctx *perf);
+
+static inline bool perf_link_is_up(struct perf_peer *peer)
+{
+	u64 link;
+
+	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
+	return !!(link & BIT_ULL_MASK(peer->pidx));
+}
+
+static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
+			      u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+	int try;
+	u32 sts;
+
+	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
+
+	/*
+	 * Perform predefined number of attempts before give up.
+	 * We are sending the data to the port specific scratchpad, so
+	 * to prevent a multi-port access race-condition. Additionally
+	 * there is no need in local locking since only thread-safe
+	 * service work is using this method.
+	 */
+	for (try = 0; try < MSG_TRIES; try++) {
+		if (!perf_link_is_up(peer))
+			return -ENOLINK;
+
+		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
+					 PERF_SPAD_CMD(perf->gidx));
+		if (le32_to_cpu(sts) != PERF_CMD_INVAL) {
+			usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
+			continue;
+		}
+
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_LDATA(perf->gidx),
+				    cpu_to_le32(lower_32_bits(data)));
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_HDATA(perf->gidx),
+				    cpu_to_le32(upper_32_bits(data)));
+		mmiowb();
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_CMD(perf->gidx),
+				    cpu_to_le32(cmd));
+		mmiowb();
+		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
+
+		dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
+			PERF_SPAD_NOTIFY(peer->gidx));
+
+		break;
+	}
+
+	return try < MSG_TRIES ? 0 : -EAGAIN;
+}
+
+static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
+			      enum perf_cmd *cmd, u64 *data)
+{
+	struct perf_peer *peer;
+	u32 val;
+
+	ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
+
+	/*
+	 * We start scanning all over, since cleared DB may have been set
+	 * by any peer. Yes, it makes peer with smaller index being
+	 * serviced with greater priority, but it's convenient for spad
+	 * and message code unification and simplicity.
+	 */
+	for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
+		peer = &perf->peers[*pidx];
+
+		if (!perf_link_is_up(peer))
+			continue;
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
+		val = le32_to_cpu(val);
+		if (val == PERF_CMD_INVAL)
+			continue;
+
+		*cmd = val;
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
+		*data = le32_to_cpu(val);
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
+		*data |= (u64)le32_to_cpu(val) << 32;
+
+		/* Next command can be retrieved from now */
+		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
+			       cpu_to_le32(PERF_CMD_INVAL));
+
+		dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
+
+		return 0;
+	}
+
+	return -ENODATA;
+}
+
+static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
+			     u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+	int try, ret;
+	u64 outbits;
+
+	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
+
+	/*
+	 * Perform predefined number of attempts before give up. Message
+	 * registers are free of race-condition problem when accessed
+	 * from different ports, so we don't need splitting registers
+	 * by global device index. We also won't have local locking,
+	 * since the method is used from service work only.
+	 */
+	outbits = ntb_msg_outbits(perf->ntb);
+	for (try = 0; try < MSG_TRIES; try++) {
+		if (!perf_link_is_up(peer))
+			return -ENOLINK;
+
+		ret = ntb_msg_clear_sts(perf->ntb, outbits);
+		if (ret)
+			return ret;
+
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
+			      cpu_to_le32(lower_32_bits(data)));
+
+		if (ntb_msg_read_sts(perf->ntb) & outbits) {
+			usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
+			continue;
+		}
+
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
+			      cpu_to_le32(upper_32_bits(data)));
+		mmiowb();
+
+		/* This call shall trigger peer message event */
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD,
+			      cpu_to_le32(cmd));
+
+		break;
+	}
+
+	return try < MSG_TRIES ? 0 : -EAGAIN;
+}
+
+static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
+			     enum perf_cmd *cmd, u64 *data)
+{
+	u64 inbits;
+	u32 val;
+
+	inbits = ntb_msg_inbits(perf->ntb);
+
+	if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
+		return -ENODATA;
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
+	*cmd = le32_to_cpu(val);
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
+	*data = le32_to_cpu(val);
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
+	*data |= (u64)le32_to_cpu(val) << 32;
+
+	/* Next command can be retrieved from now */
+	ntb_msg_clear_sts(perf->ntb, inbits);
+
+	dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
+
+	return 0;
+}
+
+static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+
+	if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
+		return perf->cmd_send(peer, cmd, data);
+
+	dev_err(&perf->ntb->dev, "Send invalid command\n");
+	return -EINVAL;
+}
+
+static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
+{
+	switch (cmd) {
+	case PERF_CMD_SSIZE:
+	case PERF_CMD_RSIZE:
+	case PERF_CMD_SXLAT:
+	case PERF_CMD_RXLAT:
+	case PERF_CMD_CLEAR:
+		break;
+	default:
+		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
+		return -EINVAL;
+	}
+
+	/* No need of memory barrier, since bit ops have invernal lock */
+	set_bit(cmd, &peer->sts);
+
+	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
+
+	(void)queue_work(system_highpri_wq, &peer->service);
+
+	return 0;
+}
+
+static int perf_cmd_recv(struct perf_ctx *perf)
+{
+	struct perf_peer *peer;
+	int ret, pidx, cmd;
+	u64 data;
+
+	while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
+		peer = &perf->peers[pidx];
+
+		switch (cmd) {
+		case PERF_CMD_SSIZE:
+			peer->inbuf_size = data;
+			return perf_cmd_exec(peer, PERF_CMD_RSIZE);
+		case PERF_CMD_SXLAT:
+			peer->outbuf_xlat = data;
+			return perf_cmd_exec(peer, PERF_CMD_RXLAT);
+		default:
+			dev_err(&perf->ntb->dev, "Recv invalid command\n");
+			return -EINVAL;
+		}
+	}
+
+	/* Return 0 if no data left to process, otherwise an error */
+	return ret == -ENODATA ? 0 : ret;
+}
+
 static void perf_link_event(void *ctx)
 {
 	struct perf_ctx *perf = ctx;
+	struct perf_peer *peer;
+	bool lnk_up;
+	int pidx;
 
-	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) {
-		schedule_delayed_work(&perf->link_work, 2*HZ);
-	} else {
-		dev_dbg(&perf->ntb->pdev->dev, "link down\n");
+	for (pidx = 0; pidx < perf->pcnt; pidx++) {
+		peer = &perf->peers[pidx];
 
-		if (!perf->link_is_up)
-			cancel_delayed_work_sync(&perf->link_work);
+		lnk_up = perf_link_is_up(peer);
 
-		perf->link_is_up = false;
+		if (lnk_up &&
+		    !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
+			perf_cmd_exec(peer, PERF_CMD_SSIZE);
+		} else if (!lnk_up &&
+			   test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
+			perf_cmd_exec(peer, PERF_CMD_CLEAR);
+		}
 	}
 }
 
 static void perf_db_event(void *ctx, int vec)
 {
 	struct perf_ctx *perf = ctx;
-	u64 db_bits, db_mask;
 
-	db_mask = ntb_db_vector_mask(perf->ntb, vec);
-	db_bits = ntb_db_read(perf->ntb);
+	dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
+		ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
+
+	/* Just receive all available commands */
+	(void)perf_cmd_recv(perf);
+}
+
+static void perf_msg_event(void *ctx)
+{
+	struct perf_ctx *perf = ctx;
+
+	dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
+		ntb_msg_read_sts(perf->ntb));
 
-	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
-		vec, db_mask, db_bits);
+	/* Messages are only sent one-by-one */
+	(void)perf_cmd_recv(perf);
 }
 
 static const struct ntb_ctx_ops perf_ops = {
 	.link_event = perf_link_event,
 	.db_event = perf_db_event,
+	.msg_event = perf_msg_event
 };
 
-static void perf_copy_callback(void *data)
+static void perf_free_outbuf(struct perf_peer *peer)
+{
+	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
+}
+
+static int perf_setup_outbuf(struct perf_peer *peer)
 {
-	struct pthr_ctx *pctx = data;
+	struct perf_ctx *perf = peer->perf;
+	int ret;
+
+	/* Outbuf size can be unaligned due to custom max_mw_size */
+	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
+				    peer->outbuf_xlat, peer->outbuf_size);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
+		return ret;
+	}
+
+	/* Initialization is finally done */
+	set_bit(PERF_STS_DONE, &peer->sts);
 
-	atomic_dec(&pctx->dma_sync);
+	return 0;
 }
 
-static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst,
-			 char *src, size_t size)
+static void perf_free_inbuf(struct perf_peer *peer)
 {
-	struct perf_ctx *perf = pctx->perf;
-	struct dma_async_tx_descriptor *txd;
-	struct dma_chan *chan = pctx->dma_chan;
-	struct dma_device *device;
-	struct dmaengine_unmap_data *unmap;
-	dma_cookie_t cookie;
-	size_t src_off, dst_off;
-	struct perf_mw *mw = &perf->mw;
-	void __iomem *vbase;
-	void __iomem *dst_vaddr;
-	dma_addr_t dst_phys;
-	int retries = 0;
+	if (!peer->inbuf)
+		return;
 
-	if (!use_dma) {
-		memcpy_toio(dst, src, size);
-		return size;
+	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
+	dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,
+			  peer->inbuf, peer->inbuf_xlat);
+	peer->inbuf = NULL;
+}
+
+static int perf_setup_inbuf(struct perf_peer *peer)
+{
+	resource_size_t xlat_align, size_align, size_max;
+	struct perf_ctx *perf = peer->perf;
+	int ret;
+
+	/* Get inbound MW parameters */
+	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
+			       &xlat_align, &size_align, &size_max);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
+		return ret;
 	}
 
-	if (!chan) {
-		dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
+	if (peer->inbuf_size > size_max) {
+		dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
+			&peer->inbuf_size, &size_max);
 		return -EINVAL;
 	}
 
-	device = chan->device;
-	src_off = (uintptr_t)src & ~PAGE_MASK;
-	dst_off = (uintptr_t __force)dst & ~PAGE_MASK;
+	peer->inbuf_size = round_up(peer->inbuf_size, size_align);
 
-	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
-		return -ENODEV;
-
-	vbase = mw->vbase;
-	dst_vaddr = dst;
-	dst_phys = mw->phys_addr + (dst_vaddr - vbase);
+	perf_free_inbuf(peer);
 
-	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
-	if (!unmap)
+	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
+					 &peer->inbuf_xlat, GFP_KERNEL);
+	if (!peer->inbuf) {
+		dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
+			&peer->inbuf_size);
 		return -ENOMEM;
+	}
+	if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
+		dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
+		goto err_free_inbuf;
+	}
 
-	unmap->len = size;
-	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
-				      src_off, size, DMA_TO_DEVICE);
-	if (dma_mapping_error(device->dev, unmap->addr[0]))
-		goto err_get_unmap;
+	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
+			       peer->inbuf_xlat, peer->inbuf_size);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
+		goto err_free_inbuf;
+	}
 
-	unmap->to_cnt = 1;
+	/*
+	 * We submit inbuf xlat transmission cmd for execution here to follow
+	 * the code architecture, even though this method is called from service
+	 * work itself so the command will be executed right after it returns.
+	 */
+	(void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
 
-	do {
-		txd = device->device_prep_dma_memcpy(chan, dst_phys,
-						     unmap->addr[0],
-						     size, DMA_PREP_INTERRUPT);
-		if (!txd) {
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(DMA_OUT_RESOURCE_TO);
-		}
-	} while (!txd && (++retries < DMA_RETRIES));
+	return 0;
 
-	if (!txd) {
-		pctx->dma_prep_err++;
-		goto err_get_unmap;
-	}
+err_free_inbuf:
+	perf_free_inbuf(peer);
 
-	txd->callback = perf_copy_callback;
-	txd->callback_param = pctx;
-	dma_set_unmap(txd, unmap);
+	return ret;
+}
 
-	cookie = dmaengine_submit(txd);
-	if (dma_submit_error(cookie))
-		goto err_set_unmap;
+static void perf_service_work(struct work_struct *work)
+{
+	struct perf_peer *peer = to_peer_service(work);
 
-	dmaengine_unmap_put(unmap);
+	if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
+		perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
 
-	atomic_inc(&pctx->dma_sync);
-	dma_async_issue_pending(chan);
+	if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
+		perf_setup_inbuf(peer);
 
-	return size;
+	if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
+		perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
 
-err_set_unmap:
-	dmaengine_unmap_put(unmap);
-err_get_unmap:
-	dmaengine_unmap_put(unmap);
-	return 0;
-}
+	if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
+		perf_setup_outbuf(peer);
 
-static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src,
-			  u64 buf_size, u64 win_size, u64 total)
-{
-	int chunks, total_chunks, i;
-	int copied_chunks = 0;
-	u64 copied = 0, result;
-	char __iomem *tmp = dst;
-	u64 perf, diff_us;
-	ktime_t kstart, kstop, kdiff;
-	unsigned long last_sleep = jiffies;
-
-	chunks = div64_u64(win_size, buf_size);
-	total_chunks = div64_u64(total, buf_size);
-	kstart = ktime_get();
-
-	for (i = 0; i < total_chunks; i++) {
-		result = perf_copy(pctx, tmp, src, buf_size);
-		copied += result;
-		copied_chunks++;
-		if (copied_chunks == chunks) {
-			tmp = dst;
-			copied_chunks = 0;
-		} else
-			tmp += buf_size;
-
-		/* Probably should schedule every 5s to prevent soft hang. */
-		if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
-			last_sleep = jiffies;
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(1);
+	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
+		clear_bit(PERF_STS_DONE, &peer->sts);
+		if (test_bit(0, &peer->perf->busy_flag) &&
+		    peer == peer->perf->test_peer) {
+			dev_warn(&peer->perf->ntb->dev,
+				"Freeing while test on-fly\n");
+			perf_terminate_test(peer->perf);
 		}
+		perf_free_outbuf(peer);
+		perf_free_inbuf(peer);
+	}
+}
 
-		if (unlikely(kthread_should_stop()))
-			break;
+static int perf_init_service(struct perf_ctx *perf)
+{
+	u64 mask;
+
+	if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {
+		dev_err(&perf->ntb->dev, "Not enough memory windows\n");
+		return -EINVAL;
 	}
 
-	if (use_dma) {
-		pr_debug("%s: All DMA descriptors submitted\n", current->comm);
-		while (atomic_read(&pctx->dma_sync) != 0) {
-			if (kthread_should_stop())
-				break;
-			msleep(20);
-		}
+	if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
+		perf->cmd_send = perf_msg_cmd_send;
+		perf->cmd_recv = perf_msg_cmd_recv;
+
+		dev_dbg(&perf->ntb->dev, "Message service initialized\n");
+
+		return 0;
 	}
 
-	kstop = ktime_get();
-	kdiff = ktime_sub(kstop, kstart);
-	diff_us = ktime_to_us(kdiff);
+	dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
 
-	pr_debug("%s: copied %llu bytes\n", current->comm, copied);
+	mask = GENMASK_ULL(perf->pcnt, 0);
+	if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
+	    (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
+		perf->cmd_send = perf_spad_cmd_send;
+		perf->cmd_recv = perf_spad_cmd_recv;
 
-	pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us);
+		dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
 
-	perf = div64_u64(copied, diff_us);
+		return 0;
+	}
 
-	pr_debug("%s: MBytes/s: %llu\n", current->comm, perf);
+	dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
 
-	pctx->copied = copied;
-	pctx->diff_us = diff_us;
+	dev_err(&perf->ntb->dev, "Command services unsupported\n");
 
-	return 0;
+	return -EINVAL;
 }
 
-static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
+static int perf_enable_service(struct perf_ctx *perf)
 {
-	/* Is the channel required to be on the same node as the device? */
-	if (!on_node)
-		return true;
+	u64 mask, incmd_bit;
+	int ret, sidx, scnt;
 
-	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
-}
+	mask = ntb_db_valid_mask(perf->ntb);
+	(void)ntb_db_set_mask(perf->ntb, mask);
 
-static int ntb_perf_thread(void *data)
-{
-	struct pthr_ctx *pctx = data;
-	struct perf_ctx *perf = pctx->perf;
-	struct pci_dev *pdev = perf->ntb->pdev;
-	struct perf_mw *mw = &perf->mw;
-	char __iomem *dst;
-	u64 win_size, buf_size, total;
-	void *src;
-	int rc, node, i;
-	struct dma_chan *dma_chan = NULL;
+	ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
+	if (ret)
+		return ret;
 
-	pr_debug("kthread %s starting...\n", current->comm);
+	if (perf->cmd_send == perf_msg_cmd_send) {
+		u64 inbits, outbits;
 
-	node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE;
+		inbits = ntb_msg_inbits(perf->ntb);
+		outbits = ntb_msg_outbits(perf->ntb);
+		(void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
 
-	if (use_dma && !pctx->dma_chan) {
-		dma_cap_mask_t dma_mask;
+		incmd_bit = BIT_ULL(__ffs64(inbits));
+		ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
 
-		dma_cap_zero(dma_mask);
-		dma_cap_set(DMA_MEMCPY, dma_mask);
-		dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
-					       (void *)(unsigned long)node);
-		if (!dma_chan) {
-			pr_warn("%s: cannot acquire DMA channel, quitting\n",
-				current->comm);
-			return -ENODEV;
-		}
-		pctx->dma_chan = dma_chan;
+		dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
+	} else {
+		scnt = ntb_spad_count(perf->ntb);
+		for (sidx = 0; sidx < scnt; sidx++)
+			ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
+		incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
+		ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
+
+		dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
+	}
+	if (ret) {
+		ntb_clear_ctx(perf->ntb);
+		return ret;
 	}
 
-	for (i = 0; i < MAX_SRCS; i++) {
-		pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
-		if (!pctx->srcs[i]) {
-			rc = -ENOMEM;
-			goto err;
-		}
+	ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	/* Might be not necessary */
+	ntb_link_event(perf->ntb);
+
+	return 0;
+}
+
+static void perf_disable_service(struct perf_ctx *perf)
+{
+	int pidx;
+
+	ntb_link_disable(perf->ntb);
+
+	if (perf->cmd_send == perf_msg_cmd_send) {
+		u64 inbits;
+
+		inbits = ntb_msg_inbits(perf->ntb);
+		(void)ntb_msg_set_mask(perf->ntb, inbits);
+	} else {
+		(void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
 	}
 
-	win_size = mw->phys_size;
-	buf_size = 1ULL << seg_order;
-	total = 1ULL << run_order;
+	ntb_clear_ctx(perf->ntb);
 
-	if (buf_size > MAX_TEST_SIZE)
-		buf_size = MAX_TEST_SIZE;
+	for (pidx = 0; pidx < perf->pcnt; pidx++)
+		perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
 
-	dst = (char __iomem *)mw->vbase;
+	for (pidx = 0; pidx < perf->pcnt; pidx++)
+		flush_work(&perf->peers[pidx].service);
+}
 
-	atomic_inc(&perf->tsync);
-	while (atomic_read(&perf->tsync) != perf->perf_threads)
-		schedule();
+/*==============================================================================
+ *                      Performance measuring work-thread
+ *==============================================================================
+ */
 
-	src = pctx->srcs[pctx->src_idx];
-	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);
+static void perf_dma_copy_callback(void *data)
+{
+	struct perf_thread *pthr = data;
 
-	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);
+	atomic_dec(&pthr->dma_sync);
+	wake_up(&pthr->dma_wait);
+}
 
-	atomic_dec(&perf->tsync);
+static int perf_copy_chunk(struct perf_thread *pthr,
+			   void __iomem *dst, void *src, size_t len)
+{
+	struct dma_async_tx_descriptor *tx;
+	struct dmaengine_unmap_data *unmap;
+	struct device *dma_dev;
+	int try = 0, ret = 0;
 
-	if (rc < 0) {
-		pr_err("%s: failed\n", current->comm);
-		rc = -ENXIO;
-		goto err;
+	if (!use_dma) {
+		memcpy_toio(dst, src, len);
+		goto ret_check_tsync;
 	}
 
-	for (i = 0; i < MAX_SRCS; i++) {
-		kfree(pctx->srcs[i]);
-		pctx->srcs[i] = NULL;
+	dma_dev = pthr->dma_chan->device->dev;
+
+	if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
+				 offset_in_page(dst), len))
+		return -EIO;
+
+	unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
+	if (!unmap)
+		return -ENOMEM;
+
+	unmap->len = len;
+	unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
+		offset_in_page(src), len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dma_dev, unmap->addr[0])) {
+		ret = -EIO;
+		goto err_free_resource;
 	}
+	unmap->to_cnt = 1;
 
-	atomic_inc(&perf->tdone);
-	wake_up(pctx->wq);
-	rc = 0;
-	goto done;
+	unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
+		offset_in_page(dst), len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dma_dev, unmap->addr[1])) {
+		ret = -EIO;
+		goto err_free_resource;
+	}
+	unmap->from_cnt = 1;
 
-err:
-	for (i = 0; i < MAX_SRCS; i++) {
-		kfree(pctx->srcs[i]);
-		pctx->srcs[i] = NULL;
+	do {
+		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
+			unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+		if (!tx)
+			msleep(DMA_MDELAY);
+	} while (!tx && (try++ < DMA_TRIES));
+
+	if (!tx) {
+		ret = -EIO;
+		goto err_free_resource;
 	}
 
-	if (dma_chan) {
-		dma_release_channel(dma_chan);
-		pctx->dma_chan = NULL;
+	tx->callback = perf_dma_copy_callback;
+	tx->callback_param = pthr;
+	dma_set_unmap(tx, unmap);
+
+	if (dma_submit_error(dmaengine_submit(tx))) {
+		dmaengine_unmap_put(unmap);
+		goto err_free_resource;
 	}
 
-done:
-	/* Wait until we are told to stop */
-	for (;;) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		if (kthread_should_stop())
-			break;
-		schedule();
+	dmaengine_unmap_put(unmap);
+
+	atomic_inc(&pthr->dma_sync);
+	dma_async_issue_pending(pthr->dma_chan);
+
+ret_check_tsync:
+	return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
+
+err_free_resource:
+	dmaengine_unmap_put(unmap);
+
+	return ret;
+}
+
+static bool perf_dma_filter(struct dma_chan *chan, void *data)
+{
+	struct perf_ctx *perf = data;
+	int node;
+
+	node = dev_to_node(&perf->ntb->dev);
+
+	return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
+}
+
+static int perf_init_test(struct perf_thread *pthr)
+{
+	struct perf_ctx *perf = pthr->perf;
+	dma_cap_mask_t dma_mask;
+
+	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
+				 dev_to_node(&perf->ntb->dev));
+	if (!pthr->src)
+		return -ENOMEM;
+
+	get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
+
+	if (!use_dma)
+		return 0;
+
+	dma_cap_zero(dma_mask);
+	dma_cap_set(DMA_MEMCPY, dma_mask);
+	pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
+	if (!pthr->dma_chan) {
+		dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
+			pthr->tidx);
+		atomic_dec(&perf->tsync);
+		wake_up(&perf->twait);
+		kfree(pthr->src);
+		return -ENODEV;
 	}
-	__set_current_state(TASK_RUNNING);
 
-	return rc;
+	atomic_set(&pthr->dma_sync, 0);
+
+	return 0;
 }
 
-static void perf_free_mw(struct perf_ctx *perf)
+static int perf_run_test(struct perf_thread *pthr)
 {
-	struct perf_mw *mw = &perf->mw;
-	struct pci_dev *pdev = perf->ntb->pdev;
+	struct perf_peer *peer = pthr->perf->test_peer;
+	struct per