RDMA/nes: Add support for MPAv2 Enhanced RDMA Negotiation

This patch adds support for Enhanced RDMA Connection Establishment
(draft-ietf-storm-mpa-peer-connect-06), aka MPAv2.  Details of draft
can be obtained from:
<http://www.ietf.org/id/draft-ietf-storm-mpa-peer-connect-06.txt>

For backwards compatibility, the MPAv2 enabled driver reverts to MPAv1
if the remote node doesn't support MPAv2.

Signed-off-by: Tatyana Nikolova <Tatyana.E.Nikolova@intel.com>
Signed-off-by: Faisal Latif <Faisal.Latif@intel.com>
Reviewed-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>

3 files changed, 703 insertions, 475 deletions

diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index c118663e4437..7dc43ea7d4e7 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -77,26 +77,19 @@ atomic_t cm_nodes_destroyed;
 atomic_t cm_accel_dropped_pkts;
 atomic_t cm_resets_recvd;
 
-static inline int mini_cm_accelerated(struct nes_cm_core *,
-	struct nes_cm_node *);
-static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *,
-	struct nes_vnic *, struct nes_cm_info *);
+static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *);
+static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *);
 static int mini_cm_del_listen(struct nes_cm_core *, struct nes_cm_listener *);
-static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *,
-	struct nes_vnic *, u16, void *, struct nes_cm_info *);
+static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *, struct nes_vnic *, u16, void *, struct nes_cm_info *);
 static int mini_cm_close(struct nes_cm_core *, struct nes_cm_node *);
-static int mini_cm_accept(struct nes_cm_core *, struct ietf_mpa_frame *,
-	struct nes_cm_node *);
-static int mini_cm_reject(struct nes_cm_core *, struct ietf_mpa_frame *,
-	struct nes_cm_node *);
-static int mini_cm_recv_pkt(struct nes_cm_core *, struct nes_vnic *,
-	struct sk_buff *);
+static int mini_cm_accept(struct nes_cm_core *, struct nes_cm_node *);
+static int mini_cm_reject(struct nes_cm_core *, struct nes_cm_node *);
+static int mini_cm_recv_pkt(struct nes_cm_core *, struct nes_vnic *, struct sk_buff *);
 static int mini_cm_dealloc_core(struct nes_cm_core *);
 static int mini_cm_get(struct nes_cm_core *);
 static int mini_cm_set(struct nes_cm_core *, u32, u32);
 
-static void form_cm_frame(struct sk_buff *, struct nes_cm_node *,
-	void *, u32, void *, u32, u8);
+static void form_cm_frame(struct sk_buff *, struct nes_cm_node *, void *, u32, void *, u32, u8);
 static int add_ref_cm_node(struct nes_cm_node *);
 static int rem_ref_cm_node(struct nes_cm_core *, struct nes_cm_node *);
 
@@ -111,16 +104,14 @@ static int send_syn(struct nes_cm_node *, u32, struct sk_buff *);
 static int send_reset(struct nes_cm_node *, struct sk_buff *);
 static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb);
 static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb);
-static void process_packet(struct nes_cm_node *, struct sk_buff *,
-	struct nes_cm_core *);
+static void process_packet(struct nes_cm_node *, struct sk_buff *, struct nes_cm_core *);
 
 static void active_open_err(struct nes_cm_node *, struct sk_buff *, int);
 static void passive_open_err(struct nes_cm_node *, struct sk_buff *, int);
 static void cleanup_retrans_entry(struct nes_cm_node *);
 static void handle_rcv_mpa(struct nes_cm_node *, struct sk_buff *);
 static void free_retrans_entry(struct nes_cm_node *cm_node);
-static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph,
-	struct sk_buff *skb, int optionsize, int passive);
+static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, struct sk_buff *skb, int optionsize, int passive);
 
 /* CM event handler functions */
 static void cm_event_connected(struct nes_cm_event *);
@@ -130,6 +121,12 @@ static void cm_event_mpa_req(struct nes_cm_event *);
 static void cm_event_mpa_reject(struct nes_cm_event *);
 static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node);
 
+/* MPA build functions */
+static int cm_build_mpa_frame(struct nes_cm_node *, u8 **, u16 *, u8 *, u8);
+static void build_mpa_v2(struct nes_cm_node *, void *, u8);
+static void build_mpa_v1(struct nes_cm_node *, void *, u8);
+static void build_rdma0_msg(struct nes_cm_node *, struct nes_qp **);
+
 static void print_core(struct nes_cm_core *core);
 
 /* External CM API Interface */
@@ -163,8 +160,8 @@ atomic_t cm_rejects;
 /**
  * create_event
  */
-static struct nes_cm_event *create_event(struct nes_cm_node *cm_node,
-		enum nes_cm_event_type type)
+static struct nes_cm_event *create_event(struct nes_cm_node *	cm_node,
+					 enum nes_cm_event_type type)
 {
 	struct nes_cm_event *event;
 
@@ -186,10 +183,10 @@ static struct nes_cm_event *create_event(struct nes_cm_node *cm_node,
 	event->cm_info.cm_id = cm_node->cm_id;
 
 	nes_debug(NES_DBG_CM, "cm_node=%p Created event=%p, type=%u, "
-		"dst_addr=%08x[%x], src_addr=%08x[%x]\n",
-		cm_node, event, type, event->cm_info.loc_addr,
-		event->cm_info.loc_port, event->cm_info.rem_addr,
-		event->cm_info.rem_port);
+		  "dst_addr=%08x[%x], src_addr=%08x[%x]\n",
+		  cm_node, event, type, event->cm_info.loc_addr,
+		  event->cm_info.loc_port, event->cm_info.rem_addr,
+		  event->cm_info.rem_port);
 
 	nes_cm_post_event(event);
 	return event;
@@ -201,14 +198,19 @@ static struct nes_cm_event *create_event(struct nes_cm_node *cm_node,
  */
 static int send_mpa_request(struct nes_cm_node *cm_node, struct sk_buff *skb)
 {
+	u8 start_addr = 0;
+	u8 *start_ptr = &start_addr;
+	u8 **start_buff = &start_ptr;
+	u16 buff_len = 0;
+
 	if (!skb) {
 		nes_debug(NES_DBG_CM, "skb set to NULL\n");
 		return -1;
 	}
 
 	/* send an MPA Request frame */
-	form_cm_frame(skb, cm_node, NULL, 0, &cm_node->mpa_frame,
-			cm_node->mpa_frame_size, SET_ACK);
+	cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REQUEST);
+	form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK);
 
 	return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
 }
@@ -217,7 +219,11 @@ static int send_mpa_request(struct nes_cm_node *cm_node, struct sk_buff *skb)
 
 static int send_mpa_reject(struct nes_cm_node *cm_node)
 {
-	struct sk_buff  *skb = NULL;
+	struct sk_buff *skb = NULL;
+	u8 start_addr = 0;
+	u8 *start_ptr = &start_addr;
+	u8 **start_buff = &start_ptr;
+	u16 buff_len = 0;
 
 	skb = dev_alloc_skb(MAX_CM_BUFFER);
 	if (!skb) {
@@ -226,8 +232,8 @@ static int send_mpa_reject(struct nes_cm_node *cm_node)
 	}
 
 	/* send an MPA reject frame */
-	form_cm_frame(skb, cm_node, NULL, 0, &cm_node->mpa_frame,
-			cm_node->mpa_frame_size, SET_ACK | SET_FIN);
+	cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REPLY);
+	form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK | SET_FIN);
 
 	cm_node->state = NES_CM_STATE_FIN_WAIT1;
 	return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0);
@@ -239,24 +245,31 @@ static int send_mpa_reject(struct nes_cm_node *cm_node)
  * IETF MPA frame
  */
 static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type,
-		u32 len)
+		     u32 len)
 {
-	struct ietf_mpa_frame *mpa_frame;
+	struct ietf_mpa_v1 *mpa_frame;
+	struct ietf_mpa_v2 *mpa_v2_frame;
+	struct ietf_rtr_msg *rtr_msg;
+	int mpa_hdr_len;
+	int priv_data_len;
 
 	*type = NES_MPA_REQUEST_ACCEPT;
 
 	/* assume req frame is in tcp data payload */
-	if (len < sizeof(struct ietf_mpa_frame)) {
+	if (len < sizeof(struct ietf_mpa_v1)) {
 		nes_debug(NES_DBG_CM, "The received ietf buffer was too small (%x)\n", len);
 		return -EINVAL;
 	}
 
-	mpa_frame = (struct ietf_mpa_frame *)buffer;
-	cm_node->mpa_frame_size = ntohs(mpa_frame->priv_data_len);
+	/* points to the beginning of the frame, which could be MPA V1 or V2 */
+	mpa_frame = (struct ietf_mpa_v1 *)buffer;
+	mpa_hdr_len = sizeof(struct ietf_mpa_v1);
+	priv_data_len = ntohs(mpa_frame->priv_data_len);
+
 	/* make sure mpa private data len is less than 512 bytes */
-	if (cm_node->mpa_frame_size > IETF_MAX_PRIV_DATA_LEN) {
+	if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
 		nes_debug(NES_DBG_CM, "The received Length of Private"
-			" Data field exceeds 512 octets\n");
+			  " Data field exceeds 512 octets\n");
 		return -EINVAL;
 	}
 	/*
@@ -264,11 +277,22 @@ static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type,
 	 * received MPA version and MPA key information
 	 *
 	 */
-	if (mpa_frame->rev != mpa_version) {
+	if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
+		nes_debug(NES_DBG_CM, "The received mpa version"
+			  " is not supported\n");
+		return -EINVAL;
+	}
+	/*
+	* backwards compatibility only
+	*/
+	if (mpa_frame->rev > cm_node->mpa_frame_rev) {
 		nes_debug(NES_DBG_CM, "The received mpa version"
-				" can not be interoperated\n");
+			" can not be interoperated\n");
 		return -EINVAL;
+	} else {
+		cm_node->mpa_frame_rev = mpa_frame->rev;
 	}
+
 	if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) {
 		if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) {
 			nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n");
@@ -281,25 +305,75 @@ static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type,
 		}
 	}
 
-	if (cm_node->mpa_frame_size + sizeof(struct ietf_mpa_frame) != len) {
+
+	if (priv_data_len + mpa_hdr_len != len) {
 		nes_debug(NES_DBG_CM, "The received ietf buffer was not right"
-				" complete (%x + %x != %x)\n",
-				cm_node->mpa_frame_size,
-				(u32)sizeof(struct ietf_mpa_frame), len);
+			" complete (%x + %x != %x)\n",
+			priv_data_len, mpa_hdr_len, len);
 		return -EINVAL;
 	}
 	/* make sure it does not exceed the max size */
 	if (len > MAX_CM_BUFFER) {
 		nes_debug(NES_DBG_CM, "The received ietf buffer was too large"
-				" (%x + %x != %x)\n",
-				cm_node->mpa_frame_size,
-				(u32)sizeof(struct ietf_mpa_frame), len);
+			" (%x + %x != %x)\n",
+			priv_data_len, mpa_hdr_len, len);
 		return -EINVAL;
 	}
 
+	cm_node->mpa_frame_size = priv_data_len;
+
+	switch (mpa_frame->rev) {
+	case IETF_MPA_V2: {
+		u16 ird_size;
+		u16 ord_size;
+		mpa_v2_frame = (struct ietf_mpa_v2 *)buffer;
+		mpa_hdr_len += IETF_RTR_MSG_SIZE;
+		cm_node->mpa_frame_size -= IETF_RTR_MSG_SIZE;
+		rtr_msg = &mpa_v2_frame->rtr_msg;
+
+		/* parse rtr message */
+		rtr_msg->ctrl_ird = ntohs(rtr_msg->ctrl_ird);
+		rtr_msg->ctrl_ord = ntohs(rtr_msg->ctrl_ord);
+		ird_size = rtr_msg->ctrl_ird & IETF_NO_IRD_ORD;
+		ord_size = rtr_msg->ctrl_ord & IETF_NO_IRD_ORD;
+
+		if (!(rtr_msg->ctrl_ird & IETF_PEER_TO_PEER)) {
+			/* send reset */
+			return -EINVAL;
+		}
+
+		if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) {
+			/* responder */
+			if (cm_node->ord_size > ird_size)
+				cm_node->ord_size = ird_size;
+		} else {
+			/* initiator */
+			if (cm_node->ord_size > ird_size)
+				cm_node->ord_size = ird_size;
+
+			if (cm_node->ird_size < ord_size) {
+				/* no resources available */
+				/* send terminate message */
+				return -EINVAL;
+			}
+		}
+
+		if (rtr_msg->ctrl_ord & IETF_RDMA0_READ) {
+			cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
+		} else if (rtr_msg->ctrl_ord & IETF_RDMA0_WRITE) {
+			cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
+		} else {        /* Not supported RDMA0 operation */
+			return -EINVAL;
+		}
+		break;
+	}
+	case IETF_MPA_V1:
+	default:
+		break;
+	}
+
 	/* copy entire MPA frame to our cm_node's frame */
-	memcpy(cm_node->mpa_frame_buf, buffer + sizeof(struct ietf_mpa_frame),
-			cm_node->mpa_frame_size);
+	memcpy(cm_node->mpa_frame_buf, buffer + mpa_hdr_len, cm_node->mpa_frame_size);
 
 	if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
 		*type = NES_MPA_REQUEST_REJECT;
@@ -312,8 +386,8 @@ static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type,
  * node info to build.
  */
 static void form_cm_frame(struct sk_buff *skb,
-	struct nes_cm_node *cm_node, void *options, u32 optionsize,
-	void *data, u32 datasize, u8 flags)
+			  struct nes_cm_node *cm_node, void *options, u32 optionsize,
+			  void *data, u32 datasize, u8 flags)
 {
 	struct tcphdr *tcph;
 	struct iphdr *iph;
@@ -322,14 +396,14 @@ static void form_cm_frame(struct sk_buff *skb,
 	u16 packetsize = sizeof(*iph);
 
 	packetsize += sizeof(*tcph);
-	packetsize +=  optionsize + datasize;
+	packetsize += optionsize + datasize;
 
+	skb_trim(skb, 0);
 	memset(skb->data, 0x00, ETH_HLEN + sizeof(*iph) + sizeof(*tcph));
 
-	skb->len = 0;
 	buf = skb_put(skb, packetsize + ETH_HLEN);
 
-	ethh = (struct ethhdr *) buf;
+	ethh = (struct ethhdr *)buf;
 	buf += ETH_HLEN;
 
 	iph = (struct iphdr *)buf;
@@ -337,7 +411,7 @@ static void form_cm_frame(struct sk_buff *skb,
 	tcph = (struct tcphdr *)buf;
 	skb_reset_mac_header(skb);
 	skb_set_network_header(skb, ETH_HLEN);
-	skb_set_transport_header(skb, ETH_HLEN+sizeof(*iph));
+	skb_set_transport_header(skb, ETH_HLEN + sizeof(*iph));
 	buf += sizeof(*tcph);
 
 	skb->ip_summed = CHECKSUM_PARTIAL;
@@ -350,14 +424,14 @@ static void form_cm_frame(struct sk_buff *skb,
 	ethh->h_proto = htons(0x0800);
 
 	iph->version = IPVERSION;
-	iph->ihl = 5;		/* 5 * 4Byte words, IP headr len */
+	iph->ihl = 5;           /* 5 * 4Byte words, IP headr len */
 	iph->tos = 0;
 	iph->tot_len = htons(packetsize);
 	iph->id = htons(++cm_node->tcp_cntxt.loc_id);
 
 	iph->frag_off = htons(0x4000);
 	iph->ttl = 0x40;
-	iph->protocol = 0x06;	/* IPPROTO_TCP */
+	iph->protocol = 0x06;   /* IPPROTO_TCP */
 
 	iph->saddr = htonl(cm_node->loc_addr);
 	iph->daddr = htonl(cm_node->rem_addr);
@@ -370,14 +444,16 @@ static void form_cm_frame(struct sk_buff *skb,
 		cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
 		tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
 		tcph->ack = 1;
-	} else
+	} else {
 		tcph->ack_seq = 0;
+	}
 
 	if (flags & SET_SYN) {
 		cm_node->tcp_cntxt.loc_seq_num++;
 		tcph->syn = 1;
-	} else
+	} else {
 		cm_node->tcp_cntxt.loc_seq_num += datasize;
+	}
 
 	if (flags & SET_FIN) {
 		cm_node->tcp_cntxt.loc_seq_num++;
@@ -398,10 +474,8 @@ static void form_cm_frame(struct sk_buff *skb,
 
 	skb_shinfo(skb)->nr_frags = 0;
 	cm_packets_created++;
-
 }
 
-
 /**
  * print_core - dump a cm core
  */
@@ -413,7 +487,7 @@ static void print_core(struct nes_cm_core *core)
 		return;
 	nes_debug(NES_DBG_CM, "---------------------------------------------\n");
 
-	nes_debug(NES_DBG_CM, "State         : %u \n",  core->state);
+	nes_debug(NES_DBG_CM, "State         : %u \n", core->state);
 
 	nes_debug(NES_DBG_CM, "Listen Nodes  : %u \n", atomic_read(&core->listen_node_cnt));
 	nes_debug(NES_DBG_CM, "Active Nodes  : %u \n", atomic_read(&core->node_cnt));
@@ -423,6 +497,147 @@ static void print_core(struct nes_cm_core *core)
 	nes_debug(NES_DBG_CM, "-------------- end core ---------------\n");
 }
 
+/**
+ * cm_build_mpa_frame - build a MPA V1 frame or MPA V2 frame
+ */
+static int cm_build_mpa_frame(struct nes_cm_node *cm_node, u8 **start_buff,
+			      u16 *buff_len, u8 *pci_mem, u8 mpa_key)
+{
+	int ret = 0;
+
+	*start_buff = (pci_mem) ? pci_mem : &cm_node->mpa_frame_buf[0];
+
+	switch (cm_node->mpa_frame_rev) {
+	case IETF_MPA_V1:
+		*start_buff = (u8 *)*start_buff + sizeof(struct ietf_rtr_msg);
+		*buff_len = sizeof(struct ietf_mpa_v1) + cm_node->mpa_frame_size;
+		build_mpa_v1(cm_node, *start_buff, mpa_key);
+		break;
+	case IETF_MPA_V2:
+		*buff_len = sizeof(struct ietf_mpa_v2) + cm_node->mpa_frame_size;
+		build_mpa_v2(cm_node, *start_buff, mpa_key);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/**
+ * build_mpa_v2 - build a MPA V2 frame
+ */
+static void build_mpa_v2(struct nes_cm_node *cm_node,
+			 void *start_addr, u8 mpa_key)
+{
+	struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr;
+	struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
+
+	/* initialize the upper 5 bytes of the frame */
+	build_mpa_v1(cm_node, start_addr, mpa_key);
+	mpa_frame->flags |= IETF_MPA_V2_FLAG; /* set a bit to indicate MPA V2 */
+	mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE);
+
+	/* initialize RTR msg */
+	rtr_msg->ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
+			    IETF_NO_IRD_ORD : cm_node->ird_size;
+	rtr_msg->ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
+			    IETF_NO_IRD_ORD : cm_node->ord_size;
+
+	rtr_msg->ctrl_ird |= IETF_PEER_TO_PEER;
+	rtr_msg->ctrl_ird |= IETF_FLPDU_ZERO_LEN;
+
+	switch (mpa_key) {
+	case MPA_KEY_REQUEST:
+		rtr_msg->ctrl_ord |= IETF_RDMA0_WRITE;
+		rtr_msg->ctrl_ord |= IETF_RDMA0_READ;
+		break;
+	case MPA_KEY_REPLY:
+		switch (cm_node->send_rdma0_op) {
+		case SEND_RDMA_WRITE_ZERO:
+			rtr_msg->ctrl_ord |= IETF_RDMA0_WRITE;
+			break;
+		case SEND_RDMA_READ_ZERO:
+			rtr_msg->ctrl_ord |= IETF_RDMA0_READ;
+			break;
+		}
+	}
+	rtr_msg->ctrl_ird = htons(rtr_msg->ctrl_ird);
+	rtr_msg->ctrl_ord = htons(rtr_msg->ctrl_ord);
+}
+
+/**
+ * build_mpa_v1 - build a MPA V1 frame
+ */
+static void build_mpa_v1(struct nes_cm_node *cm_node, void *start_addr, u8 mpa_key)
+{
+	struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr;
+
+	switch (mpa_key) {
+	case MPA_KEY_REQUEST:
+		memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
+		break;
+	case MPA_KEY_REPLY:
+		memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
+		break;
+	}
+	mpa_frame->flags = IETF_MPA_FLAGS_CRC;
+	mpa_frame->rev = cm_node->mpa_frame_rev;
+	mpa_frame->priv_data_len = htons(cm_node->mpa_frame_size);
+}
+
+static void build_rdma0_msg(struct nes_cm_node *cm_node, struct nes_qp **nesqp_addr)
+{
+	u64 u64temp;
+	struct nes_qp *nesqp = *nesqp_addr;
+	struct nes_hw_qp_wqe *wqe = &nesqp->hwqp.sq_vbase[0];
+
+	u64temp = (unsigned long)nesqp;
+	u64temp |= NES_SW_CONTEXT_ALIGN >> 1;
+	set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, u64temp);
+
+	wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = 0;
+	wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = 0;
+
+	switch (cm_node->send_rdma0_op) {
+	case SEND_RDMA_WRITE_ZERO:
+		nes_debug(NES_DBG_CM, "Sending first write.\n");
+		wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] =
+			cpu_to_le32(NES_IWARP_SQ_OP_RDMAW);
+		wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = 0;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = 0;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 0;
+		break;
+
+	case SEND_RDMA_READ_ZERO:
+	default:
+		if (cm_node->send_rdma0_op != SEND_RDMA_READ_ZERO) {
+			printk(KERN_ERR "%s[%u]: Unsupported RDMA0 len operation=%u\n",
+				 __func__, __LINE__, cm_node->send_rdma0_op);
+			WARN_ON(1);
+		}
+		nes_debug(NES_DBG_CM, "Sending first rdma operation.\n");
+		wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] =
+			cpu_to_le32(NES_IWARP_SQ_OP_RDMAR);
+		wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = 1;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = 0;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 0;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = 1;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 1;
+		break;
+	}
+
+	if (nesqp->sq_kmapped) {
+		nesqp->sq_kmapped = 0;
+		kunmap(nesqp->page);
+	}
+
+	/*use the reserved spot on the WQ for the extra first WQE*/
+	nesqp->nesqp_context->ird_ord_sizes &= cpu_to_le32(~(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT |
+							     NES_QPCONTEXT_ORDIRD_WRPDU |
+							     NES_QPCONTEXT_ORDIRD_ALSMM));
+	nesqp->skip_lsmm = 1;
+	nesqp->hwqp.sq_tail = 0;
+}
 
 /**
  * schedule_nes_timer
@@ -430,10 +645,10 @@ static void print_core(struct nes_cm_core *core)
  *			rem_ref_cm_node(cm_core, cm_node);add_ref_cm_node(cm_node);
  */
 int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
-		enum nes_timer_type type, int send_retrans,
-		int close_when_complete)
+		       enum nes_timer_type type, int send_retrans,
+		       int close_when_complete)
 {
-	unsigned long  flags;
+	unsigned long flags;
 	struct nes_cm_core *cm_core = cm_node->cm_core;
 	struct nes_timer_entry *new_send;
 	int ret = 0;
@@ -454,7 +669,7 @@ int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
 	new_send->close_when_complete = close_when_complete;
 
 	if (type == NES_TIMER_TYPE_CLOSE) {
-		new_send->timetosend += (HZ/10);
+		new_send->timetosend += (HZ / 10);
 		if (cm_node->recv_entry) {
 			kfree(new_send);
 			WARN_ON(1);
@@ -475,7 +690,7 @@ int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb,
 		ret = nes_nic_cm_xmit(new_send->skb, cm_node->netdev);
 		if (ret != NETDEV_TX_OK) {
 			nes_debug(NES_DBG_CM, "Error sending packet %p "
-				"(jiffies = %lu)\n", new_send, jiffies);
+				  "(jiffies = %lu)\n", new_send, jiffies);
 			new_send->timetosend = jiffies;
 			ret = NETDEV_TX_OK;
 		} else {
@@ -504,6 +719,7 @@ static void nes_retrans_expired(struct nes_cm_node *cm_node)
 	struct iw_cm_id *cm_id = cm_node->cm_id;
 	enum nes_cm_node_state state = cm_node->state;
 	cm_node->state = NES_CM_STATE_CLOSED;
+
 	switch (state) {
 	case NES_CM_STATE_SYN_RCVD:
 	case NES_CM_STATE_CLOSING:
@@ -536,10 +752,10 @@ static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node)
 		spin_lock_irqsave(&nesqp->lock, qplockflags);
 		if (nesqp->cm_id) {
 			nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, "
-				"refcount = %d: HIT A "
-				"NES_TIMER_TYPE_CLOSE with something "
-				"to do!!!\n", nesqp->hwqp.qp_id, cm_id,
-				atomic_read(&nesqp->refcount));
+				  "refcount = %d: HIT A "
+				  "NES_TIMER_TYPE_CLOSE with something "
+				  "to do!!!\n", nesqp->hwqp.qp_id, cm_id,
+				  atomic_read(&nesqp->refcount));
 			nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
 			nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;
 			nesqp->ibqp_state = IB_QPS_ERR;
@@ -548,10 +764,10 @@ static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node)
 		} else {
 			spin_unlock_irqrestore(&nesqp->lock, qplockflags);
 			nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, "
-				"refcount = %d: HIT A "
-				"NES_TIMER_TYPE_CLOSE with nothing "
-				"to do!!!\n", nesqp->hwqp.qp_id, cm_id,
-				atomic_read(&nesqp->refcount));
+				  "refcount = %d: HIT A "
+				  "NES_TIMER_TYPE_CLOSE with nothing "
+				  "to do!!!\n", nesqp->hwqp.qp_id, cm_id,
+				  atomic_read(&nesqp->refcount));
 		}
 	} else if (rem_node) {
 		/* TIME_WAIT state */
@@ -580,11 +796,12 @@ static void nes_cm_timer_tick(unsigned long pass)
 	int ret = NETDEV_TX_OK;
 
 	struct list_head timer_list;
+
 	INIT_LIST_HEAD(&timer_list);
 	spin_lock_irqsave(&cm_core->ht_lock, flags);
 
 	list_for_each_safe(list_node, list_core_temp,
-				&cm_core->connected_nodes) {
+			   &cm_core->connected_nodes) {
 		cm_node = container_of(list_node, struct nes_cm_node, list);
 		if ((cm_node->recv_entry) || (cm_node->send_entry)) {
 			add_ref_cm_node(cm_node);
@@ -595,18 +812,19 @@ static void nes_cm_timer_tick(unsigned long pass)
 
 	list_for_each_safe(list_node, list_core_temp, &timer_list) {
 		cm_node = container_of(list_node, struct nes_cm_node,
-					timer_entry);
+				       timer_entry);
 		recv_entry = cm_node->recv_entry;
 
 		if (recv_entry) {
 			if (time_after(recv_entry->timetosend, jiffies)) {
 				if (nexttimeout > recv_entry->timetosend ||
-						!settimer) {
+				    !settimer) {
 					nexttimeout = recv_entry->timetosend;
 					settimer = 1;
 				}
-			} else
+			} else {
 				handle_recv_entry(cm_node, 1);
+			}
 		}
 
 		spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
@@ -617,8 +835,8 @@ static void nes_cm_timer_tick(unsigned long pass)
 			if (time_after(send_entry->timetosend, jiffies)) {
 				if (cm_node->state != NES_CM_STATE_TSA) {
 					if ((nexttimeout >
-						send_entry->timetosend) ||
-						!settimer) {
+					     send_entry->timetosend) ||
+					    !settimer) {
 						nexttimeout =
 							send_entry->timetosend;
 						settimer = 1;
@@ -630,13 +848,13 @@ static void nes_cm_timer_tick(unsigned long pass)
 			}
 
 			if ((cm_node->state == NES_CM_STATE_TSA) ||
-				(cm_node->state == NES_CM_STATE_CLOSED)) {
+			    (cm_node->state == NES_CM_STATE_CLOSED)) {
 				free_retrans_entry(cm_node);
 				break;
 			}
 
 			if (!send_entry->retranscount ||
-				!send_entry->retrycount) {
+			    !send_entry->retrycount) {
 				cm_packets_dropped++;
 				free_retrans_entry(cm_node);
 
@@ -645,28 +863,28 @@ static void nes_cm_timer_tick(unsigned long pass)
 				nes_retrans_expired(cm_node);
 				cm_node->state = NES_CM_STATE_CLOSED;
 				spin_lock_irqsave(&cm_node->retrans_list_lock,
-					flags);
+						  flags);
 				break;
 			}
 			atomic_inc(&send_entry->skb->users);
 			cm_packets_retrans++;
 			nes_debug(NES_DBG_CM, "Retransmitting send_entry %p "
-				"for node %p, jiffies = %lu, time to send = "
-				"%lu, retranscount = %u, send_entry->seq_num = "
-				"0x%08X, cm_node->tcp_cntxt.rem_ack_num = "
-				"0x%08X\n", send_entry, cm_node, jiffies,
-				send_entry->timetosend,
-				send_entry->retranscount,
-				send_entry->seq_num,
-				cm_node->tcp_cntxt.rem_ack_num);
+				  "for node %p, jiffies = %lu, time to send = "
+				  "%lu, retranscount = %u, send_entry->seq_num = "
+				  "0x%08X, cm_node->tcp_cntxt.rem_ack_num = "
+				  "0x%08X\n", send_entry, cm_node, jiffies,
+				  send_entry->timetosend,
+				  send_entry->retranscount,
+				  send_entry->seq_num,
+				  cm_node->tcp_cntxt.rem_ack_num);
 
 			spin_unlock_irqrestore(&cm_node->retrans_list_lock,
-				flags);
+					       flags);
 			ret = nes_nic_cm_xmit(send_entry->skb, cm_node->netdev);
 			spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
 			if (ret != NETDEV_TX_OK) {
 				nes_debug(NES_DBG_CM, "rexmit failed for "
-					"node=%p\n", cm_node);
+					  "node=%p\n", cm_node);
 				cm_packets_bounced++;
 				send_entry->retrycount--;
 				nexttimeout = jiffies + NES_SHORT_TIME;
@@ -676,18 +894,18 @@ static void nes_cm_timer_tick(unsigned long pass)
 				cm_packets_sent++;
 			}
 			nes_debug(NES_DBG_CM, "Packet Sent: retrans count = "
-				"%u, retry count = %u.\n",
-				send_entry->retranscount,
-				send_entry->retrycount);
+				  "%u, retry count = %u.\n",
+				  send_entry->retranscount,
+				  send_entry->retrycount);
 			if (send_entry->send_retrans) {
 				send_entry->retranscount--;
 				timetosend = (NES_RETRY_TIMEOUT <<
-					(NES_DEFAULT_RETRANS - send_entry->retranscount));
+					      (NES_DEFAULT_RETRANS - send_entry->retranscount));
 
 				send_entry->timetosend = jiffies +
-					min(timetosend, NES_MAX_TIMEOUT);
+							 min(timetosend, NES_MAX_TIMEOUT);
 				if (nexttimeout > send_entry->timetosend ||
-					!settimer) {
+				    !settimer) {
 					nexttimeout = send_entry->timetosend;
 					settimer = 1;
 				}
@@ -696,11 +914,11 @@ static void nes_cm_timer_tick(unsigned long pass)
 				close_when_complete =
 					send_entry->close_when_complete;
 				nes_debug(NES_DBG_CM, "cm_node=%p state=%d\n",
-					cm_node, cm_node->state);
+					  cm_node, cm_node->state);
 				free_retrans_entry(cm_node);
 				if (close_when_complete)
 					rem_ref_cm_node(cm_node->cm_core,
-						cm_node);
+							cm_node);
 			}
 		} while (0);
 
@@ -710,7 +928,7 @@ static void nes_cm_timer_tick(unsigned long pass)
 
 	if (settimer) {
 		if (!timer_pending(&cm_core->tcp_timer)) {
-			cm_core->tcp_timer.expires  = nexttimeout;
+			cm_core->tcp_timer.expires = nexttimeout;
 			add_timer(&cm_core->tcp_timer);
 		}
 	}
@@ -721,13 +939,13 @@ static void nes_cm_timer_tick(unsigned long pass)
  * send_syn
  */
 static int send_syn(struct nes_cm_node *cm_node, u32 sendack,
-	struct sk_buff *skb)
+		    struct sk_buff *skb)
 {
 	int ret;
 	int flags = SET_SYN;
 	char optionsbuffer[sizeof(struct option_mss) +
-		sizeof(struct option_windowscale) + sizeof(struct option_base) +
-		TCP_OPTIONS_PADDING];
+			   sizeof(struct option_windowscale) + sizeof(struct option_base) +
+			   TCP_OPTIONS_PADDING];
 
 	int optionssize = 0;
 	/* Sending MSS option */
@@ -854,7 +1072,7 @@ static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb)
  * find_node - find a cm node that matches the reference cm node
  */
 static struct nes_cm_node *find_node(struct nes_cm_core *cm_core,
-		u16 rem_port, nes_addr_t rem_addr, u16 loc_port, nes_addr_t loc_addr)
+				     u16 rem_port, nes_addr_t rem_addr, u16 loc_port, nes_addr_t loc_addr)
 {
 	unsigned long flags;
 	struct list_head *hte;
@@ -868,12 +1086,12 @@ static struct nes_cm_node *find_node(struct nes_cm_core *cm_core,
 	list_for_each_entry(cm_node, hte, list) {
 		/* compare quad, return node handle if a match */
 		nes_debug(NES_DBG_CM, "finding node %x:%x =? %x:%x ^ %x:%x =? %x:%x\n",
-				cm_node->loc_addr, cm_node->loc_port,
-				loc_addr, loc_port,
-				cm_node->rem_addr, cm_node->rem_port,
-				rem_addr, rem_port);
+			  cm_node->loc_addr, cm_node->loc_port,
+			  loc_addr, loc_port,
+			  cm_node->rem_addr, cm_node->rem_port,
+			  rem_addr, rem_port);
 		if ((cm_node->loc_addr == loc_addr) && (cm_node->loc_port == loc_port) &&
-				(cm_node->rem_addr == rem_addr) && (cm_node->rem_port == rem_port)) {
+		    (cm_node->rem_addr == rem_addr) && (cm_node->rem_port == rem_port)) {
 			add_ref_cm_node(cm_node);
 			spin_unlock_irqrestore(&cm_core->ht_lock, flags);
 			return cm_node;
@@ -890,7 +1108,7 @@ static struct nes_cm_node *find_node(struct nes_cm_core *cm_core,
  * find_listener - find a cm node listening on this addr-port pair
  */
 static struct nes_cm_listener *find_listener(struct nes_cm_core *cm_core,
-		nes_addr_t dst_addr, u16 dst_port, enum nes_cm_listener_state listener_state)
+					     nes_addr_t dst_addr, u16 dst_port, enum nes_cm_listener_state listener_state)
 {
 	unsigned long flags;
 	struct nes_cm_listener *listen_node;
@@ -900,9 +1118,9 @@ static struct nes_cm_listener *find_listener(struct nes_cm_core *cm_core,
 	list_for_each_entry(listen_node, &cm_core->listen_list.list, list) {
 		/* compare node pair, return node handle if a match */
 		if (((listen_node->loc_addr == dst_addr) ||
-				listen_node->loc_addr == 0x00000000) &&
-				(listen_node->loc_port == dst_port) &&
-				(listener_state & listen_node->listener_state)) {
+		     listen_node->loc_addr == 0x00000000) &&
+		    (listen_node->loc_port == dst_port) &&
+		    (listener_state & listen_node->listener_state)) {
 			atomic_inc(&listen_node->ref_count);
 			spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 			return listen_node;
@@ -927,7 +1145,7 @@ static int add_hte_node(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node
 		return -EINVAL;
 
 	nes_debug(NES_DBG_CM, "Adding Node %p to Active Connection HT\n",
-		cm_node);
+		  cm_node);
 
 	spin_lock_irqsave(&cm_core->ht_lock, flags);
 
@@ -946,7 +1164,7 @@ static int add_hte_node(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node
  * mini_cm_dec_refcnt_listen
  */
 static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
-	struct nes_cm_listener *listener, int free_hanging_nodes)
+				     struct nes_cm_listener *listener, int free_hanging_nodes)
 {
 	int ret = -EINVAL;
 	int err = 0;
@@ -957,8 +1175,8 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 	struct list_head reset_list;
 
 	nes_debug(NES_DBG_CM, "attempting listener= %p free_nodes= %d, "
-		"refcnt=%d\n", listener, free_hanging_nodes,
-		atomic_read(&listener->ref_count));
+		  "refcnt=%d\n", listener, free_hanging_nodes,
+		  atomic_read(&listener->ref_count));
 	/* free non-accelerated child nodes for this listener */
 	INIT_LIST_HEAD(&reset_list);
 	if (free_hanging_nodes) {
@@ -966,7 +1184,7 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 		list_for_each_safe(list_pos, list_temp,
 				   &g_cm_core->connected_nodes) {
 			cm_node = container_of(list_pos, struct nes_cm_node,
-				list);
+					       list);
 			if ((cm_node->listener == listener) &&
 			    (!cm_node->accelerated)) {
 				add_ref_cm_node(cm_node);
@@ -978,7 +1196,7 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 
 	list_for_each_safe(list_pos, list_temp, &reset_list) {
 		cm_node = container_of(list_pos, struct nes_cm_node,
-				reset_entry);
+				       reset_entry);
 		{
 			struct nes_cm_node *loopback = cm_node->loopbackpartner;
 			enum nes_cm_node_state old_state;
@@ -990,7 +1208,7 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 					err = send_reset(cm_node, NULL);
 					if (err) {
 						cm_node->state =
-							 NES_CM_STATE_CLOSED;
+							NES_CM_STATE_CLOSED;
 						WARN_ON(1);
 					} else {
 						old_state = cm_node->state;
@@ -1035,10 +1253,9 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 
 		spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 
-		if (listener->nesvnic) {
+		if (listener->nesvnic)
 			nes_manage_apbvt(listener->nesvnic, listener->loc_port,
-					PCI_FUNC(listener->nesvnic->nesdev->pcidev->devfn), NES_MANAGE_APBVT_DEL);
-		}
+					 PCI_FUNC(listener->nesvnic->nesdev->pcidev->devfn), NES_MANAGE_APBVT_DEL);
 
 		nes_debug(NES_DBG_CM, "destroying listener (%p)\n", listener);
 
@@ -1052,8 +1269,8 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
 	if (listener) {
 		if (atomic_read(&listener->pend_accepts_cnt) > 0)
 			nes_debug(NES_DBG_CM, "destroying listener (%p)"
-					" with non-zero pending accepts=%u\n",
-					listener, atomic_read(&listener->pend_accepts_cnt));
+				  " with non-zero pending accepts=%u\n",
+				  listener, atomic_read(&listener->pend_accepts_cnt));
 	}
 
 	return ret;
@@ -1064,7 +1281,7 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
  * mini_cm_del_listen
  */
 static int mini_cm_del_listen(struct nes_cm_core *cm_core,
-		struct nes_cm_listener *listener)
+			      struct nes_cm_listener *listener)
 {
 	listener->listener_state = NES_CM_LISTENER_PASSIVE_STATE;
 	listener->cm_id = NULL; /* going to be destroyed pretty soon */
@@ -1076,9 +1293,10 @@ static int mini_cm_del_listen(struct nes_cm_core *cm_core,
  * mini_cm_accelerated
  */
 static inline int mini_cm_accelerated(struct nes_cm_core *cm_core,
-		struct nes_cm_node *cm_node)
+				      struct nes_cm_node *cm_node)
 {
 	u32 was_timer_set;
+
 	cm_node->accelerated = 1;
 
 	if (cm_node->accept_pend) {
@@ -1112,7 +1330,7 @@ static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpi
 	rt = ip_route_output(&init_net, htonl(dst_ip), 0, 0, 0);
 	if (IS_ERR(rt)) {
 		printk(KERN_ERR "%s: ip_route_output_key failed for 0x%08X\n",
-				__func__, dst_ip);
+		       __func__, dst_ip);
 		return rc;
 	}
 
@@ -1130,7 +1348,7 @@ static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpi
 
 			if (arpindex >= 0) {
 				if (!memcmp(nesadapter->arp_table[arpindex].mac_addr,
-							neigh->ha, ETH_ALEN)){
+					    neigh->ha, ETH_ALEN)) {
 					/* Mac address same as in nes_arp_table */
 					neigh_release(neigh);
 					ip_rt_put(rt);
@@ -1138,8 +1356,8 @@ static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpi
 				}
 
 				nes_manage_arp_cache(nesvnic->netdev,
-						nesadapter->arp_table[arpindex].mac_addr,
-						dst_ip, NES_ARP_DELETE);
+						     nesadapter->arp_table[arpindex].mac_addr,
+						     dst_ip, NES_ARP_DELETE);
 			}
 
 			nes_manage_arp_cache(nesvnic->netdev, neigh->ha,
@@ -1161,8 +1379,8 @@ static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpi
  * make_cm_node - create a new instance of a cm node
  */
 static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core,
-		struct nes_vnic *nesvnic, struct nes_cm_info *cm_info,
-		struct nes_cm_listener *listener)
+					struct nes_vnic *nesvnic, struct nes_cm_info *c