crypto/chcr: move nic TLS functionality to drivers/net

This patch moves complete nic tls offload (kTLS) code from crypto
directory to drivers/net/ethernet/chelsio/inline_crypto/ch_ktls
directory. nic TLS is made a separate ULD of cxgb4.

Signed-off-by: Rohit Maheshwari <rohitm@chelsio.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

7 files changed, 0 insertions, 2305 deletions

diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index af161dab49bd..f886401af13e 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -21,14 +21,3 @@ config CRYPTO_DEV_CHELSIO
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called chcr.
-
-config CHELSIO_TLS_DEVICE
-	bool "Chelsio Inline KTLS Offload"
-	depends on CHELSIO_T4
-	depends on TLS_DEVICE
-	select CRYPTO_DEV_CHELSIO
-	default y
-	help
-	  This flag enables support for kernel tls offload over Chelsio T6
-	  crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled
-	  only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index f2e8e2fb4e60..2e5df484ab01 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -3,6 +3,3 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4
 
 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
 chcr-objs :=  chcr_core.o chcr_algo.o
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-chcr-objs += chcr_ktls.o
-#endif
diff --git a/drivers/crypto/chelsio/chcr_common.h b/drivers/crypto/chelsio/chcr_common.h
deleted file mode 100644
index 33f589cbfba1..000000000000
--- a/drivers/crypto/chelsio/chcr_common.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
-
-#ifndef __CHCR_COMMON_H__
-#define __CHCR_COMMON_H__
-
-#include "cxgb4.h"
-
-#define CHCR_MAX_SALT                      4
-#define CHCR_KEYCTX_MAC_KEY_SIZE_128       0
-#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128    0
-#define CHCR_SCMD_CIPHER_MODE_AES_GCM      2
-#define CHCR_SCMD_CIPHER_MODE_AES_CTR      3
-#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT      2
-#define CHCR_SCMD_SEQ_NO_CTRL_64BIT        3
-#define CHCR_SCMD_PROTO_VERSION_TLS        0
-#define CHCR_SCMD_PROTO_VERSION_GENERIC    4
-#define CHCR_SCMD_AUTH_MODE_GHASH          4
-#define AES_BLOCK_LEN                      16
-
-enum chcr_state {
-	CHCR_INIT = 0,
-	CHCR_ATTACH,
-	CHCR_DETACH,
-};
-
-struct chcr_dev {
-	spinlock_t lock_chcr_dev; /* chcr dev structure lock */
-	enum chcr_state state;
-	atomic_t inflight;
-	int wqretry;
-	struct delayed_work detach_work;
-	struct completion detach_comp;
-	unsigned char tx_channel_id;
-};
-
-struct uld_ctx {
-	struct list_head entry;
-	struct cxgb4_lld_info lldi;
-	struct chcr_dev dev;
-};
-
-struct ktls_key_ctx {
-	__be32 ctx_hdr;
-	u8 salt[CHCR_MAX_SALT];
-	__be64 iv_to_auth;
-	unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE +
-			  TLS_CIPHER_AES_GCM_256_TAG_SIZE];
-};
-
-/* Crypto key context */
-#define KEY_CONTEXT_CTX_LEN_S           24
-#define KEY_CONTEXT_CTX_LEN_V(x)        ((x) << KEY_CONTEXT_CTX_LEN_S)
-
-#define KEY_CONTEXT_SALT_PRESENT_S      10
-#define KEY_CONTEXT_SALT_PRESENT_V(x)   ((x) << KEY_CONTEXT_SALT_PRESENT_S)
-#define KEY_CONTEXT_SALT_PRESENT_F      KEY_CONTEXT_SALT_PRESENT_V(1U)
-
-#define KEY_CONTEXT_VALID_S     0
-#define KEY_CONTEXT_VALID_V(x)  ((x) << KEY_CONTEXT_VALID_S)
-#define KEY_CONTEXT_VALID_F     KEY_CONTEXT_VALID_V(1U)
-
-#define KEY_CONTEXT_CK_SIZE_S           6
-#define KEY_CONTEXT_CK_SIZE_V(x)        ((x) << KEY_CONTEXT_CK_SIZE_S)
-
-#define KEY_CONTEXT_MK_SIZE_S           2
-#define KEY_CONTEXT_MK_SIZE_V(x)        ((x) << KEY_CONTEXT_MK_SIZE_S)
-
-#define KEY_CONTEXT_OPAD_PRESENT_S      11
-#define KEY_CONTEXT_OPAD_PRESENT_V(x)   ((x) << KEY_CONTEXT_OPAD_PRESENT_S)
-#define KEY_CONTEXT_OPAD_PRESENT_F      KEY_CONTEXT_OPAD_PRESENT_V(1U)
-
-#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \
-		htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \
-		      KEY_CONTEXT_CK_SIZE_V(ck_size) | \
-		      KEY_CONTEXT_VALID_F | \
-		      KEY_CONTEXT_SALT_PRESENT_F | \
-		      KEY_CONTEXT_CTX_LEN_V((ctx_len)))
-
-struct uld_ctx *assign_chcr_device(void);
-
-static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q,
-				     void *pos, int length)
-{
-	int left = (void *)q->stat - pos;
-	u64 *p;
-
-	if (likely(length <= left)) {
-		memcpy(pos, src, length);
-		pos += length;
-	} else {
-		memcpy(pos, src, left);
-		memcpy(q->desc, src + left, length - left);
-		pos = (void *)q->desc + (length - left);
-	}
-	/* 0-pad to multiple of 16 */
-	p = PTR_ALIGN(pos, 8);
-	if ((uintptr_t)p & 8) {
-		*p = 0;
-		return p + 1;
-	}
-	return p;
-}
-
-static inline unsigned int chcr_txq_avail(const struct sge_txq *q)
-{
-	return q->size - 1 - q->in_use;
-}
-
-static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n)
-{
-	q->in_use += n;
-	q->pidx += n;
-	if (q->pidx >= q->size)
-		q->pidx -= q->size;
-}
-
-static inline void chcr_eth_txq_stop(struct sge_eth_txq *q)
-{
-	netif_tx_stop_queue(q->txq);
-	q->q.stops++;
-}
-
-static inline unsigned int chcr_sgl_len(unsigned int n)
-{
-	n--;
-	return (3 * n) / 2 + (n & 1) + 2;
-}
-
-static inline unsigned int chcr_flits_to_desc(unsigned int n)
-{
-	WARN_ON(n > SGE_MAX_WR_LEN / 8);
-	return DIV_ROUND_UP(n, 8);
-}
-#endif /* __CHCR_COMMON_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index b3570b41a737..40d51d2bd935 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -33,19 +33,8 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input);
 static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
 static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
 
-#if defined(CONFIG_CHELSIO_TLS_DEVICE)
-static const struct tlsdev_ops chcr_ktls_ops = {
-	.tls_dev_add = chcr_ktls_dev_add,
-	.tls_dev_del = chcr_ktls_dev_del,
-};
-#endif
-
 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
 	[CPL_FW6_PLD] = cpl_fw6_pld_handler,
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-	[CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
-	[CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
-#endif
 };
 
 static struct cxgb4_uld_info chcr_uld_info = {
@@ -56,10 +45,6 @@ static struct cxgb4_uld_info chcr_uld_info = {
 	.add = chcr_uld_add,
 	.state_change = chcr_uld_state_change,
 	.rx_handler = chcr_uld_rx_handler,
-#if defined(CONFIG_CHELSIO_TLS_DEVICE)
-	.tx_handler = chcr_uld_tx_handler,
-	.tlsdev_ops = &chcr_ktls_ops,
-#endif
 };
 
 static void detach_work_fn(struct work_struct *work)
@@ -235,15 +220,6 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
 	return 0;
 }
 
-#if defined(CONFIG_CHELSIO_TLS_DEVICE)
-int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev)
-{
-	if (skb->decrypted)
-		return chcr_ktls_xmit(skb, dev);
-	return 0;
-}
-#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */
-
 static void chcr_detach_device(struct uld_ctx *u_ctx)
 {
 	struct chcr_dev *dev = &u_ctx->dev;
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index 81f6e61401e5..bb092b6b36b2 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -139,16 +139,4 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 		     int err);
 int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
 void chcr_add_xfrmops(const struct cxgb4_lld_info *lld);
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev);
-extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
-			     enum tls_offload_ctx_dir direction,
-			     struct tls_crypto_info *crypto_info,
-			     u32 start_offload_tcp_sn);
-extern void chcr_ktls_dev_del(struct net_device *netdev,
-			      struct tls_context *tls_ctx,
-			      enum tls_offload_ctx_dir direction);
-#endif
 #endif /* __CHCR_CORE_H__ */
diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c
deleted file mode 100644
index c5cce024886a..000000000000
--- a/drivers/crypto/chelsio/chcr_ktls.c
+++ /dev/null
@@ -1,2017 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
-
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-#include <linux/highmem.h>
-#include "chcr_ktls.h"
-#include "clip_tbl.h"
-
-static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
-/*
- * chcr_ktls_save_keys: calculate and save crypto keys.
- * @tx_info - driver specific tls info.
- * @crypto_info - tls crypto information.
- * @direction - TX/RX direction.
- * return - SUCCESS/FAILURE.
- */
-static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
-			       struct tls_crypto_info *crypto_info,
-			       enum tls_offload_ctx_dir direction)
-{
-	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
-	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
-	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
-	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
-	struct crypto_cipher *cipher;
-	unsigned char *key, *salt;
-
-	switch (crypto_info->cipher_type) {
-	case TLS_CIPHER_AES_GCM_128:
-		info_128_gcm =
-			(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
-		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-		tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
-		mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
-		tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
-		tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
-
-		ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
-		key = info_128_gcm->key;
-		salt = info_128_gcm->salt;
-		tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
-
-		/* The SCMD fields used when encrypting a full TLS
-		 * record. Its a one time calculation till the
-		 * connection exists.
-		 */
-		tx_info->scmd0_seqno_numivs =
-			SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
-			SCMD_CIPH_AUTH_SEQ_CTRL_F |
-			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
-			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
-			SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
-			SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
-			SCMD_NUM_IVS_V(1);
-
-		/* keys will be sent inline. */
-		tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
-
-		/* The SCMD fields used when encrypting a partial TLS
-		 * record (no trailer and possibly a truncated payload).
-		 */
-		tx_info->scmd0_short_seqno_numivs =
-			SCMD_CIPH_AUTH_SEQ_CTRL_F |
-			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
-			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
-			SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
-
-		tx_info->scmd0_short_ivgen_hdrlen =
-			tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
-
-		break;
-
-	default:
-		pr_err("GCM: cipher type 0x%x not supported\n",
-		       crypto_info->cipher_type);
-		ret = -EINVAL;
-		goto out;
-	}
-
-	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
-		       roundup(keylen, 16) + ghash_size;
-	/* Calculate the H = CIPH(K, 0 repeated 16 times).
-	 * It will go in key context
-	 */
-	cipher = crypto_alloc_cipher("aes", 0, 0);
-	if (IS_ERR(cipher)) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	ret = crypto_cipher_setkey(cipher, key, keylen);
-	if (ret)
-		goto out1;
-
-	memset(ghash_h, 0, ghash_size);
-	crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
-
-	/* fill the Key context */
-	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
-		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
-						 mac_key_size,
-						 key_ctx_size >> 4);
-	} else {
-		ret = -EINVAL;
-		goto out1;
-	}
-
-	memcpy(kctx->salt, salt, tx_info->salt_size);
-	memcpy(kctx->key, key, keylen);
-	memcpy(kctx->key + keylen, ghash_h, ghash_size);
-	tx_info->key_ctx_len = key_ctx_size;
-
-out1:
-	crypto_free_cipher(cipher);
-out:
-	return ret;
-}
-
-static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
-					     int new_state)
-{
-	/* This function can be called from both rx (interrupt context) and tx
-	 * queue contexts.
-	 */
-	spin_lock_bh(&tx_info->lock);
-	switch (tx_info->connection_state) {
-	case KTLS_CONN_CLOSED:
-		tx_info->connection_state = new_state;
-		break;
-
-	case KTLS_CONN_ACT_OPEN_REQ:
-		/* only go forward if state is greater than current state. */
-		if (new_state <= tx_info->connection_state)
-			break;
-		/* update to the next state and also initialize TCB */
-		tx_info->connection_state = new_state;
-		fallthrough;
-	case KTLS_CONN_ACT_OPEN_RPL:
-		/* if we are stuck in this state, means tcb init might not
-		 * received by HW, try sending it again.
-		 */
-		if (!chcr_init_tcb_fields(tx_info))
-			tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
-		break;
-
-	case KTLS_CONN_SET_TCB_REQ:
-		/* only go forward if state is greater than current state. */
-		if (new_state <= tx_info->connection_state)
-			break;
-		/* update to the next state and check if l2t_state is valid  */
-		tx_info->connection_state = new_state;
-		fallthrough;
-	case KTLS_CONN_SET_TCB_RPL:
-		/* Check if l2t state is valid, then move to ready state. */
-		if (cxgb4_check_l2t_valid(tx_info->l2te)) {
-			tx_info->connection_state = KTLS_CONN_TX_READY;
-			atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx);
-		}
-		break;
-
-	case KTLS_CONN_TX_READY:
-		/* nothing to be done here */
-		break;
-
-	default:
-		pr_err("unknown KTLS connection state\n");
-		break;
-	}
-	spin_unlock_bh(&tx_info->lock);
-
-	return tx_info->connection_state;
-}
-/*
- * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * @atid - connection active tid.
- * return - send success/failure.
- */
-static int chcr_ktls_act_open_req(struct sock *sk,
-				  struct chcr_ktls_info *tx_info,
-				  int atid)
-{
-	struct inet_sock *inet = inet_sk(sk);
-	struct cpl_t6_act_open_req *cpl6;
-	struct cpl_act_open_req *cpl;
-	struct sk_buff *skb;
-	unsigned int len;
-	int qid_atid;
-	u64 options;
-
-	len = sizeof(*cpl6);
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (unlikely(!skb))
-		return -ENOMEM;
-	/* mark it a control pkt */
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-
-	cpl6 = __skb_put_zero(skb, len);
-	cpl = (struct cpl_act_open_req *)cpl6;
-	INIT_TP_WR(cpl6, 0);
-	qid_atid = TID_QID_V(tx_info->rx_qid) |
-		   TID_TID_V(atid);
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
-	cpl->local_port = inet->inet_sport;
-	cpl->peer_port = inet->inet_dport;
-	cpl->local_ip = inet->inet_rcv_saddr;
-	cpl->peer_ip = inet->inet_daddr;
-
-	/* fill first 64 bit option field. */
-	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
-		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
-	cpl->opt0 = cpu_to_be64(options);
-
-	/* next 64 bit option field. */
-	options =
-		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
-	cpl->opt2 = htonl(options);
-
-	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
-}
-
-#if IS_ENABLED(CONFIG_IPV6)
-/*
- * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * @atid - connection active tid.
- * return - send success/failure.
- */
-static int chcr_ktls_act_open_req6(struct sock *sk,
-				   struct chcr_ktls_info *tx_info,
-				   int atid)
-{
-	struct inet_sock *inet = inet_sk(sk);
-	struct cpl_t6_act_open_req6 *cpl6;
-	struct cpl_act_open_req6 *cpl;
-	struct sk_buff *skb;
-	unsigned int len;
-	int qid_atid;
-	u64 options;
-
-	len = sizeof(*cpl6);
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (unlikely(!skb))
-		return -ENOMEM;
-	/* mark it a control pkt */
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-
-	cpl6 = __skb_put_zero(skb, len);
-	cpl = (struct cpl_act_open_req6 *)cpl6;
-	INIT_TP_WR(cpl6, 0);
-	qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
-	cpl->local_port = inet->inet_sport;
-	cpl->peer_port = inet->inet_dport;
-	cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
-	cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
-	cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
-	cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
-
-	/* first 64 bit option field. */
-	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
-		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
-	cpl->opt0 = cpu_to_be64(options);
-	/* next 64 bit option field. */
-	options =
-		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
-	cpl->opt2 = htonl(options);
-
-	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
-}
-#endif /* #if IS_ENABLED(CONFIG_IPV6) */
-
-/*
- * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP
- */
-static int chcr_setup_connection(struct sock *sk,
-				 struct chcr_ktls_info *tx_info)
-{
-	struct tid_info *t = &tx_info->adap->tids;
-	int atid, ret = 0;
-
-	atid = cxgb4_alloc_atid(t, tx_info);
-	if (atid == -1)
-		return -EINVAL;
-
-	tx_info->atid = atid;
-	tx_info->ip_family = sk->sk_family;
-
-	if (sk->sk_family == AF_INET) {
-		tx_info->ip_family = AF_INET;
-		ret = chcr_ktls_act_open_req(sk, tx_info, atid);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		if (!sk->sk_ipv6only &&
-		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
-			tx_info->ip_family = AF_INET;
-			ret = chcr_ktls_act_open_req(sk, tx_info, atid);
-		} else {
-			tx_info->ip_family = AF_INET6;
-			ret = cxgb4_clip_get(tx_info->netdev,
-					     (const u32 *)
-					     &sk->sk_v6_rcv_saddr.s6_addr,
-					     1);
-			if (ret)
-				goto out;
-			ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
-		}
-#endif
-	}
-
-	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
-	 * success, if any other return type clear atid and return that failure.
-	 */
-	if (ret) {
-		if (ret == NET_XMIT_CN)
-			ret = 0;
-		else
-			cxgb4_free_atid(t, atid);
-		goto out;
-	}
-
-	/* update the connection state */
-	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
-out:
-	return ret;
-}
-
-/*
- * chcr_set_tcb_field: update tcb fields.
- * @tx_info - driver specific tls info.
- * @word - TCB word.
- * @mask - TCB word related mask.
- * @val - TCB word related value.
- * @no_reply - set 1 if not looking for TP response.
- */
-static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
-			      u64 mask, u64 val, int no_reply)
-{
-	struct cpl_set_tcb_field *req;
-	struct sk_buff *skb;
-
-	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
-	if (!skb)
-		return -ENOMEM;
-
-	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
-	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
-	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
-				NO_REPLY_V(no_reply));
-	req->word_cookie = htons(TCB_WORD_V(word));
-	req->mask = cpu_to_be64(mask);
-	req->val = cpu_to_be64(val);
-
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-	return cxgb4_ofld_send(tx_info->netdev, skb);
-}
-
-/*
- * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP.
- */
-static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
-{
-	return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
-				  TCB_T_STATE_V(TCB_T_STATE_M),
-				  CHCR_TCB_STATE_CLOSED, 1);
-}
-
-/*
- * chcr_ktls_dev_del:  call back for tls_dev_del.
- * Remove the tid and l2t entry and close the connection.
- * it per connection basis.
- * @netdev - net device.
- * @tls_cts - tls context.
- * @direction - TX/RX crypto direction
- */
-void chcr_ktls_dev_del(struct net_device *netdev,
-		       struct tls_context *tls_ctx,
-		       enum tls_offload_ctx_dir direction)
-{
-	struct chcr_ktls_ofld_ctx_tx *tx_ctx =
-				chcr_get_ktls_tx_context(tls_ctx);
-	struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
-	struct sock *sk;
-
-	if (!tx_info)
-		return;
-	sk = tx_info->sk;
-
-	spin_lock(&tx_info->lock);
-	tx_info->connection_state = KTLS_CONN_CLOSED;
-	spin_unlock(&tx_info->lock);
-
-	/* clear l2t entry */
-	if (tx_info->l2te)
-		cxgb4_l2t_release(tx_info->l2te);
-
-#if IS_ENABLED(CONFIG_IPV6)
-	/* clear clip entry */
-	if (tx_info->ip_family == AF_INET6)
-		cxgb4_clip_release(netdev,
-				   (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8,
-				   1);
-#endif
-
-	/* clear tid */
-	if (tx_info->tid != -1) {
-		/* clear tcb state and then release tid */
-		chcr_ktls_mark_tcb_close(tx_info);
-		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
-				 tx_info->tid, tx_info->ip_family);
-	}
-
-	atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close);
-	kvfree(tx_info);
-	tx_ctx->chcr_info = NULL;
-	/* release module refcount */
-	module_put(THIS_MODULE);
-}
-
-/*
- * chcr_ktls_dev_add:  call back for tls_dev_add.
- * Create a tcb entry for TP. Also add l2t entry for the connection. And
- * generate keys & save those keys locally.
- * @netdev - net device.
- * @tls_cts - tls context.
- * @direction - TX/RX crypto direction
- * return: SUCCESS/FAILURE.
- */
-int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
-		      enum tls_offload_ctx_dir direction,
-		      struct tls_crypto_info *crypto_info,
-		      u32 start_offload_tcp_sn)
-{
-	struct tls_context *tls_ctx = tls_get_ctx(sk);
-	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
-	struct chcr_ktls_info *tx_info;
-	struct dst_entry *dst;
-	struct adapter *adap;
-	struct port_info *pi;
-	struct neighbour *n;
-	u8 daaddr[16];
-	int ret = -1;
-
-	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
-
-	pi = netdev_priv(netdev);
-	adap = pi->adapter;
-	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
-		pr_err("not expecting for RX direction\n");
-		ret = -EINVAL;
-		goto out;
-	}
-	if (tx_ctx->chcr_info) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
-	if (!tx_info) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	spin_lock_init(&tx_info->lock);
-
-	/* clear connection state */
-	spin_lock(&tx_info->lock);
-	tx_info->connection_state = KTLS_CONN_CLOSED;
-	spin_unlock(&tx_info->lock);
-
-	tx_info->sk = sk;
-	/* initialize tid and atid to -1, 0 is a also a valid id. */
-	tx_info->tid = -1;
-	tx_info->atid = -1;
-
-	tx_info->adap = adap;
-	tx_info->netdev = netdev;
-	tx_info->first_qset = pi->first_qset;
-	tx_info->tx_chan = pi->tx_chan;
-	tx_info->smt_idx = pi->smt_idx;
-	tx_info->port_id = pi->port_id;
-
-	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
-	if (unlikely(tx_info->rx_qid < 0))
-		goto out2;
-
-	tx_info->prev_seq = start_offload_tcp_sn;
-	tx_info->tcp_start_seq_number = start_offload_tcp_sn;
-
-	/* save crypto keys */
-	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
-	if (ret < 0)
-		goto out2;
-
-	/* get peer ip */
-	if (sk->sk_family == AF_INET) {
-		memcpy(daaddr, &sk->sk_daddr, 4);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		if (!sk->sk_ipv6only &&
-		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)
-			memcpy(daaddr, &sk->sk_daddr, 4);
-		else
-			memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
-#endif
-	}
-
-	/* get the l2t index */
-	dst = sk_dst_get(sk);
-	if (!dst) {
-		pr_err("DST entry not found\n");
-		goto out2;
-	}
-	n = dst_neigh_lookup(dst, daaddr);
-	if (!n || !n->dev) {
-		pr_err("neighbour not found\n");
-		dst_release(dst);
-		goto out2;
-	}
-	tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
-
-	neigh_release(n);
-	dst_release(dst);
-
-	if (!tx_info->l2te) {
-		pr_err("l2t entry not found\n");
-		goto out2;
-	}
-
-	tx_ctx->chcr_info = tx_info;
-
-	/* create a filter and call cxgb4_l2t_send to send the packet out, which
-	 * will take care of updating l2t entry in hw if not already done.
-	 */
-	ret = chcr_setup_connection(sk, tx_info);
-	if (ret)
-		goto out2;
-
-	/* Driver shouldn't be removed until any single connection exists */
-	if (!try_module_get(THIS_MODULE)) {
-		ret = -EINVAL;
-		goto out2;
-	}
-
-	atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open);
-	return 0;
-out2:
-	kvfree(tx_info);
-out:
-	atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail);
-	return ret;
-}
-
-/*
- * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
- *			  handling.
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP
- */
-static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
-{
-	int  ret = 0;
-
-	/* set tcb in offload and bypass */
-	ret =
-	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
-			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
-			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
-	if (ret)
-		return ret;
-	/* reset snd_una and snd_next fields in tcb */
-	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
-				 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
-				 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
-				 0, 1);
-	if (ret)
-		return ret;
-
-	/* reset send max */
-	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
-				 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
-				 0, 1);
-	if (ret)
-		return ret;
-
-	/* update l2t index and request for tp reply to confirm tcb is
-	 * initialised to handle tx traffic.
-	 */
-	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
-				 TCB_L2T_IX_V(TCB_L2T_IX_M),
-				 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
-	return ret;
-}
-
-/*
- * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
- */
-int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
-{
-	const struct cpl_act_open_rpl *p = (void *)input;
-	struct chcr_ktls_info *tx_info = NULL;
-	unsigned int atid, tid, status;
-	struct tid_info *t;
-
-	tid = GET_TID(p);
-	status = AOPEN_STATUS_G(ntohl(p->atid_status));
-	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
-
-	t = &adap->tids;
-	tx_info = lookup_atid(t, atid);
-
-	if (!tx_info || tx_info->atid != atid) {
-		pr_err("tx_info or atid is not correct\n");
-		return -1;
-	}
-
-	if (!status) {
-		tx_info->tid = tid;
-		cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
-
-		cxgb4_free_atid(t, atid);
-		tx_info->atid = -1;
-		/* update the connection state */
-		chcr_ktls_update_connection_state(tx_info,
-						  KTLS_CONN_ACT_OPEN_RPL);
-	}
-	return 0;
-}
-
-/*
- * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
- */
-int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
-{
-	const struct cpl_set_tcb_rpl *p = (void *)input;
-	struct chcr_ktls_info *tx_info = NULL;
-	struct tid_info *t;
-	u32 tid;
-
-	tid = GET_TID(p);
-
-	t = &adap->tids;
-	tx_info = lookup_tid(t, tid);
-	if (!tx_info || tx_info->tid != tid) {
-		pr_err("tx_info or atid is not correct\n");
-		return -1;
-	}
-	/* update the connection state */
-	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
-	return 0;
-}
-
-static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
-					u32 tid, void *pos, u16 word, u64 mask,
-					u64 val, u32 reply)
-{
-	struct cpl_set_tcb_field_core *cpl;
-	struct ulptx_idata *idata;
-	struct ulp_txpkt *txpkt;
-
-	/* ULP_TXPKT */
-	txpkt = pos;
-	txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0));
-	txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
-
-	/* ULPTX_IDATA sub-command */
-	idata = (struct ulptx_idata *)(txpkt + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
-	idata->len = htonl(sizeof(*cpl));
-	pos = idata + 1;
-
-	cpl = pos;
-	/* CPL_SET_TCB_FIELD */
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
-	cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
-			NO_REPLY_V(!reply));
-	cpl->word_cookie = htons(TCB_WORD_V(word));
-	cpl->mask = cpu_to_be64(mask);
-	cpl->val = cpu_to_be64(val);
-
-	/* ULPTX_NOOP */
-	idata = (struct ulptx_idata *)(cpl + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
-	idata->len = htonl(0);
-	pos = idata + 1;
-
-	return pos;
-}
-
-
-/*
- * chcr_write_cpl_set_tcb_ulp: update tcb values.
- * TCB is responsible to create tcp headers, so all the related values
- * should be correctly updated.
- * @tx_info - driver specific tls info.
- * @q - tx queue on which packet is going out.
- * @tid - TCB identifier.
- * @pos - current index where should we start writing.
- * @word - TCB word.
- * @mask - TCB word related mask.
- * @val - TCB word related value.
- * @reply - set 1 if looking for TP response.
- * return - next position to write.
- */
-static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
-					struct sge_eth_txq *q, u32 tid,
-					void *pos, u16 word, u64 mask,
-					u64 val, u32 reply)
-{
-	int left = (void *)q->q.stat - pos;
-
-	if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
-		if (!left) {
-			pos = q->q.desc;
-		} else {
-			u8 buf[48] = {0};
-
-			__chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word,
-						     mask, val, reply);
-
-			return chcr_copy_to_txd(buf, &q->q, pos,
-						CHCR_SET_TCB_FIELD_LEN);
-		}
-	}
-
-	pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word,
-					   mask, val, reply);
-
-	/* check again if we are at the end of the queue */
-	if (left == CHCR_SET_TCB_FIELD_LEN)
-		pos = q->q.desc;
-
-	return pos;
-}
-
-/*
- * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
- * with updated values like tcp seq, ack, window etc.
- * @tx_info - driver specific tls info.
- * @q - TX queue.
- * @tcp_seq
- * @tcp_ack
- * @tcp_win
- * return: NETDEV_TX_BUSY/NET_TX_OK.
- */
-static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
-				   struct sge_eth_txq *q, u64 tcp_seq,
-				   u64 tcp_ack, u64 tcp_win)
-{
-	bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
-	u32 len, cpl = 0, ndesc, wr_len;
-	struct fw_ulptx_wr *wr;
-	int credits;
-	void *pos;
-
-	wr_len = sizeof(*wr);
-	/* there can be max 4 cpls, check if we have enough credits */
-	len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
-	ndesc = DIV_ROUND_UP(len, 64);
-
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	/* make space for WR, we'll fill it later when we know all the cpls
-	 * being sent out and have complete length.
-	 */
-	wr = pos;
-	pos += wr_len;
-	/* update tx_max if its a re-transmit or the first wr */
-	if (first_wr || tcp_seq != tx_info->prev_seq) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_TX_MAX_W,
-						 TCB_TX_MAX_V(TCB_TX_MAX_M),
-						 TCB_TX_MAX_V(tcp_seq), 0);
-		cpl++;
-	}
-	/* reset snd una if it's a re-transmit pkt */
-	if (tcp_seq != tx_info->prev_seq) {
-		/* reset snd_una */
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_SND_UNA_RAW_W,
-						 TCB_SND_UNA_RAW_V
-						 (TCB_SND_UNA_RAW_M),
-						 TCB_SND_UNA_RAW_V(0), 0);
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo);
-		cpl++;
-	}
-	/* update ack */
-	if (first_wr || tx_info->prev_ack != tcp_ack) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_RCV_NXT_W,
-						 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
-						 TCB_RCV_NXT_V(tcp_ack), 0);
-		tx_info->prev_ack = tcp_ack;
-		cpl++;
-	}
-	/* update receive window */
-	if (first_wr || tx_info->prev_win != tcp_win) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_RCV_WND_W,
-						 TCB_RCV_WND_V(TCB_RCV_WND_M),
-						 TCB_RCV_WND_V(tcp_wi