path: root/include/rdma/rdmavt_qp.h

/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
 * Copyright(c) 2016 - 2020 Intel Corporation.
 */

#ifndef DEF_RDMAVT_INCQP_H
#define DEF_RDMAVT_INCQP_H

#include <rdma/rdma_vt.h>
#include <rdma/ib_pack.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdmavt_cq.h>
#include <rdma/rvt-abi.h>
/*
 * Atomic bit definitions for r_aflags.
 */
#define RVT_R_WRID_VALID        0
#define RVT_R_REWIND_SGE        1

/*
 * Bit definitions for r_flags.
 */
#define RVT_R_REUSE_SGE 0x01
#define RVT_R_RDMAR_SEQ 0x02
#define RVT_R_RSP_NAK   0x04
#define RVT_R_RSP_SEND  0x08
#define RVT_R_COMM_EST  0x10

/*
 * If a packet's QP[23:16] bits match this value, then it is
 * a PSM packet and the hardware will expect a KDETH header
 * following the BTH.
 */
#define RVT_KDETH_QP_PREFIX       0x80
#define RVT_KDETH_QP_SUFFIX       0xffff
#define RVT_KDETH_QP_PREFIX_MASK  0x00ff0000
#define RVT_KDETH_QP_PREFIX_SHIFT 16
#define RVT_KDETH_QP_BASE         (u32)(RVT_KDETH_QP_PREFIX << \
					RVT_KDETH_QP_PREFIX_SHIFT)
#define RVT_KDETH_QP_MAX          (u32)(RVT_KDETH_QP_BASE + RVT_KDETH_QP_SUFFIX)

/*
 * If a packet's LNH == BTH and DEST QPN[23:16] in the BTH match this
 * prefix value, then it is an AIP packet with a DETH containing the entropy
 * value in byte 4 following the BTH.
 */
#define RVT_AIP_QP_PREFIX       0x81
#define RVT_AIP_QP_SUFFIX       0xffff
#define RVT_AIP_QP_PREFIX_MASK  0x00ff0000
#define RVT_AIP_QP_PREFIX_SHIFT 16
#define RVT_AIP_QP_BASE         (u32)(RVT_AIP_QP_PREFIX << \
				      RVT_AIP_QP_PREFIX_SHIFT)
#define RVT_AIP_QPN_MAX         BIT(RVT_AIP_QP_PREFIX_SHIFT)
#define RVT_AIP_QP_MAX          (u32)(RVT_AIP_QP_BASE + RVT_AIP_QPN_MAX - 1)

/*
 * Bit definitions for s_flags.
 *
 * RVT_S_SIGNAL_REQ_WR - set if QP send WRs contain completion signaled
 * RVT_S_BUSY - send tasklet is processing the QP
 * RVT_S_TIMER - the RC retry timer is active
 * RVT_S_ACK_PENDING - an ACK is waiting to be sent after RDMA read/atomics
 * RVT_S_WAIT_FENCE - waiting for all prior RDMA read or atomic SWQEs
 *                         before processing the next SWQE
 * RVT_S_WAIT_RDMAR - waiting for a RDMA read or atomic SWQE to complete
 *                         before processing the next SWQE
 * RVT_S_WAIT_RNR - waiting for RNR timeout
 * RVT_S_WAIT_SSN_CREDIT - waiting for RC credits to process next SWQE
 * RVT_S_WAIT_DMA - waiting for send DMA queue to drain before generating
 *                  next send completion entry not via send DMA
 * RVT_S_WAIT_PIO - waiting for a send buffer to be available
 * RVT_S_WAIT_TX - waiting for a struct verbs_txreq to be available
 * RVT_S_WAIT_DMA_DESC - waiting for DMA descriptors to be available
 * RVT_S_WAIT_KMEM - waiting for kernel memory to be available
 * RVT_S_WAIT_PSN - waiting for a packet to exit the send DMA queue
 * RVT_S_WAIT_ACK - waiting for an ACK packet before sending more requests
 * RVT_S_SEND_ONE - send one packet, request ACK, then wait for ACK
 * RVT_S_ECN - a BECN was queued to the send engine
 * RVT_S_MAX_BIT_MASK - The max bit that can be used by rdmavt
 */
#define RVT_S_SIGNAL_REQ_WR	0x0001
#define RVT_S_BUSY		0x0002
#define RVT_S_TIMER		0x0004
#define RVT_S_RESP_PENDING	0x0008
#define RVT_S_ACK_PENDING	0x0010
#define RVT_S_WAIT_FENCE	0x0020
#define RVT_S_WAIT_RDMAR	0x0040
#define RVT_S_WAIT_RNR		0x0080
#define RVT_S_WAIT_SSN_CREDIT	0x0100
#define RVT_S_WAIT_DMA		0x0200
#define RVT_S_WAIT_PIO		0x0400
#define RVT_S_WAIT_TX		0x0800
#define RVT_S_WAIT_DMA_DESC	0x1000
#define RVT_S_WAIT_KMEM		0x2000
#define RVT_S_WAIT_PSN		0x4000
#define RVT_S_WAIT_ACK		0x8000
#define RVT_S_SEND_ONE		0x10000
#define RVT_S_UNLIMITED_CREDIT	0x20000
#define RVT_S_ECN		0x40000
#define RVT_S_MAX_BIT_MASK	0x800000

/*
 * Drivers should use s_flags starting with bit 31 down to the bit next to
 * RVT_S_MAX_BIT_MASK
 */

/*
 * Wait flags that would prevent any packet type from being sent.
 */
#define RVT_S_ANY_WAIT_IO \
	(RVT_S_WAIT_PIO | RVT_S_WAIT_TX | \
	 RVT_S_WAIT_DMA_DESC | RVT_S_WAIT_KMEM)

/*
 * Wait flags that would prevent send work requests from making progress.
 */
#define RVT_S_ANY_WAIT_SEND (RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | \
	RVT_S_WAIT_RNR | RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_DMA | \
	RVT_S_WAIT_PSN | RVT_S_WAIT_ACK)

#define RVT_S_ANY_WAIT (RVT_S_ANY_WAIT_IO | RVT_S_ANY_WAIT_SEND)

/* Number of bits to pay attention to in the opcode for checking qp type */
#define RVT_OPCODE_QP_MASK 0xE0

/* Flags for checking QP state (see ib_rvt_state_ops[]) */
#define RVT_POST_SEND_OK                0x01
#define RVT_POST_RECV_OK                0x02
#define RVT_PROCESS_RECV_OK             0x04
#define RVT_PROCESS_SEND_OK             0x08
#define RVT_PROCESS_NEXT_SEND_OK        0x10
#define RVT_FLUSH_SEND			0x20
#define RVT_FLUSH_RECV			0x40
#define RVT_PROCESS_OR_FLUSH_SEND \
	(RVT_PROCESS_SEND_OK | RVT_FLUSH_SEND)
#define RVT_SEND_OR_FLUSH_OR_RECV_OK \
	(RVT_PROCESS_SEND_OK | RVT_FLUSH_SEND | RVT_PROCESS_RECV_OK)

/*
 * Internal send flags
 */
#define RVT_SEND_RESERVE_USED           IB_SEND_RESERVED_START
#define RVT_SEND_COMPLETION_ONLY	(IB_SEND_RESERVED_START << 1)

/**
 * rvt_ud_wr - IB UD work plus AH cache
 * @wr: valid IB work request
 * @attr: pointer to an allocated AH attribute
 *
 * Special case the UD WR so we can keep track of the AH attributes.
 *
 * NOTE: This data structure is stricly ordered wr then attr. I.e the attr
 * MUST come after wr.  The ib_ud_wr is sized and copied in rvt_post_one_wr.
 * The copy assumes that wr is first.
 */
struct rvt_ud_wr {
	struct ib_ud_wr wr;
	struct rdma_ah_attr *attr;
};

/*
 * Send work request queue entry.
 * The size of the sg_list is determined when the QP is created and stored
 * in qp->s_max_sge.
 */
struct rvt_swqe {
	union {
		struct ib_send_wr wr;   /* don't use wr.sg_list */
		struct rvt_ud_wr ud_wr;
		struct ib_reg_wr reg_wr;
		struct ib_rdma_wr rdma_wr;
		struct ib_atomic_wr atomic_wr;
	};
	u32 psn;                /* first packet sequence number */
	u32 lpsn;               /* last packet sequence number */
	u32 ssn;                /* send sequence number */
	u32 length;             /* total length of data in sg_list */
	void *priv;             /* driver dependent field */
	struct rvt_sge sg_list[];
};

/**
 * struct rvt_krwq - kernel struct receive work request
 * @p_lock: lock to protect producer of the kernel buffer
 * @head: index of next entry to fill
 * @c_lock:lock to protect consumer of the kernel buffer
 * @tail: index of next entry to pull
 * @count: count is aproximate of total receive enteries posted
 * @rvt_rwqe: struct of receive work request queue entry
 *
 * This structure is used to contain the head pointer,
 * tail pointer and receive work queue entries for kernel
 * mode user.
 */
struct rvt_krwq {
	spinlock_t p_lock;	/* protect producer */
	u32 head;               /* new work requests posted to the head */

	/* protect consumer */
	spinlock_t c_lock ____cacheline_aligned_in_smp;
	u32 tail;               /* receives pull requests from here. */
	u32 count;		/* approx count of receive entries posted */
	struct rvt_rwqe *curr_wq;
	struct rvt_rwqe wq[];
};

/*
 * rvt_get_swqe_ah - Return the pointer to the struct rvt_ah
 * @swqe: valid Send WQE
 *
 */
static inline struct rvt_ah *rvt_get_swqe_ah(struct rvt_swqe *swqe)
{
	return ibah_to_rvtah(swqe->ud_wr.wr.ah);
}

/**
 * rvt_get_swqe_ah_attr - Return the cached ah attribute information
 * @swqe: valid Send WQE
 *
 */
static inline struct rdma_ah_attr *rvt_get_swqe_ah_attr(struct rvt_swqe *swqe)
{
	return swqe->ud_wr.attr;
}

/**
 * rvt_get_swqe_remote_qpn - Access the remote QPN value
 * @swqe: valid Send WQE
 *
 */
static inline u32 rvt_get_swqe_remote_qpn(struct rvt_swqe *swqe)
{
	return swqe->ud_wr.wr.remote_qpn;
}

/**
 * rvt_get_swqe_remote_qkey - Acces the remote qkey value
 * @swqe: valid Send WQE
 *
 */
static inline u32 rvt_get_swqe_remote_qkey(struct rvt_swqe *swqe)
{
	return swqe->ud_wr.wr.remote_qkey;
}

/**
 * rvt_get_swqe_pkey_index - Access the pkey index
 * @swqe: valid Send WQE
 *
 */
static inline u16 rvt_get_swqe_pkey_index(struct rvt_swqe *swqe)
{
	return swqe->ud_wr.wr.pkey_index;
}

struct rvt_rq {
	struct rvt_rwq *wq;
	struct rvt_krwq *kwq;
	u32 size;               /* size of RWQE array */
	u8 max_sge;
	/* protect changes in this struct */
	spinlock_t lock ____cacheline_aligned_in_smp;
};

/**
 * rvt_get_rq_count - count numbers of request work queue entries
 * in circular buffer
 * @rq: data structure for request queue entry
 * @head: head indices of the circular buffer
 * @tail: tail indices of the circular buffer
 *
 * Return - total number of entries in the Receive Queue
 */

static inline u32 rvt_get_rq_count(struct rvt_rq *rq, u32 head, u32 tail)
{
	u32 count = head - tail;

	if