diff options
author | Rohit Maheshwari <rohitm@chelsio.com> | 2020-09-10 19:51:47 +0530 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2020-09-11 17:26:39 -0700 |
commit | a8c16e8ed624f24b2b082fb9a193e0132a5fd108 (patch) | |
tree | a761730f8f6043295bcc75e4ba6a1383110eda18 /drivers/crypto | |
parent | 6b5525c86c9263e2b04e6d2d42a860212e0cf881 (diff) |
crypto/chcr: move nic TLS functionality to drivers/net
This patch moves complete nic tls offload (kTLS) code from crypto
directory to drivers/net/ethernet/chelsio/inline_crypto/ch_ktls
directory. nic TLS is made a separate ULD of cxgb4.
Signed-off-by: Rohit Maheshwari <rohitm@chelsio.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/crypto')
-rw-r--r-- | drivers/crypto/chelsio/Kconfig | 11 | ||||
-rw-r--r-- | drivers/crypto/chelsio/Makefile | 3 | ||||
-rw-r--r-- | drivers/crypto/chelsio/chcr_common.h | 135 | ||||
-rw-r--r-- | drivers/crypto/chelsio/chcr_core.c | 24 | ||||
-rw-r--r-- | drivers/crypto/chelsio/chcr_core.h | 12 | ||||
-rw-r--r-- | drivers/crypto/chelsio/chcr_ktls.c | 2017 | ||||
-rw-r--r-- | drivers/crypto/chelsio/chcr_ktls.h | 103 |
7 files changed, 0 insertions, 2305 deletions
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index af161dab49bd..f886401af13e 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -21,14 +21,3 @@ config CRYPTO_DEV_CHELSIO To compile this driver as a module, choose M here: the module will be called chcr. - -config CHELSIO_TLS_DEVICE - bool "Chelsio Inline KTLS Offload" - depends on CHELSIO_T4 - depends on TLS_DEVICE - select CRYPTO_DEV_CHELSIO - default y - help - This flag enables support for kernel tls offload over Chelsio T6 - crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled - only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index f2e8e2fb4e60..2e5df484ab01 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,6 +3,3 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o -#ifdef CONFIG_CHELSIO_TLS_DEVICE -chcr-objs += chcr_ktls.o -#endif diff --git a/drivers/crypto/chelsio/chcr_common.h b/drivers/crypto/chelsio/chcr_common.h deleted file mode 100644 index 33f589cbfba1..000000000000 --- a/drivers/crypto/chelsio/chcr_common.h +++ /dev/null @@ -1,135 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifndef __CHCR_COMMON_H__ -#define __CHCR_COMMON_H__ - -#include "cxgb4.h" - -#define CHCR_MAX_SALT 4 -#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 -#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 -#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 -#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 -#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT 2 -#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 -#define CHCR_SCMD_PROTO_VERSION_TLS 0 -#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 -#define CHCR_SCMD_AUTH_MODE_GHASH 4 -#define AES_BLOCK_LEN 16 - -enum chcr_state { - CHCR_INIT = 0, - CHCR_ATTACH, - CHCR_DETACH, -}; - -struct chcr_dev { - spinlock_t lock_chcr_dev; /* chcr dev structure lock */ - enum chcr_state state; - atomic_t inflight; - int wqretry; - struct delayed_work detach_work; - struct completion detach_comp; - unsigned char tx_channel_id; -}; - -struct uld_ctx { - struct list_head entry; - struct cxgb4_lld_info lldi; - struct chcr_dev dev; -}; - -struct ktls_key_ctx { - __be32 ctx_hdr; - u8 salt[CHCR_MAX_SALT]; - __be64 iv_to_auth; - unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE + - TLS_CIPHER_AES_GCM_256_TAG_SIZE]; -}; - -/* Crypto key context */ -#define KEY_CONTEXT_CTX_LEN_S 24 -#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) - -#define KEY_CONTEXT_SALT_PRESENT_S 10 -#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) -#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) - -#define KEY_CONTEXT_VALID_S 0 -#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) -#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) - -#define KEY_CONTEXT_CK_SIZE_S 6 -#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) - -#define KEY_CONTEXT_MK_SIZE_S 2 -#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) - -#define KEY_CONTEXT_OPAD_PRESENT_S 11 -#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) -#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) - -#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \ - htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \ - KEY_CONTEXT_CK_SIZE_V(ck_size) | \ - KEY_CONTEXT_VALID_F | \ - KEY_CONTEXT_SALT_PRESENT_F | \ - KEY_CONTEXT_CTX_LEN_V((ctx_len))) - -struct uld_ctx *assign_chcr_device(void); - -static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q, - void *pos, int length) -{ - int left = (void *)q->stat - pos; - u64 *p; - - if (likely(length <= left)) { - memcpy(pos, src, length); - pos += length; - } else { - memcpy(pos, src, left); - memcpy(q->desc, src + left, length - left); - pos = (void *)q->desc + (length - left); - } - /* 0-pad to multiple of 16 */ - p = PTR_ALIGN(pos, 8); - if ((uintptr_t)p & 8) { - *p = 0; - return p + 1; - } - return p; -} - -static inline unsigned int chcr_txq_avail(const struct sge_txq *q) -{ - return q->size - 1 - q->in_use; -} - -static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n) -{ - q->in_use += n; - q->pidx += n; - if (q->pidx >= q->size) - q->pidx -= q->size; -} - -static inline void chcr_eth_txq_stop(struct sge_eth_txq *q) -{ - netif_tx_stop_queue(q->txq); - q->q.stops++; -} - -static inline unsigned int chcr_sgl_len(unsigned int n) -{ - n--; - return (3 * n) / 2 + (n & 1) + 2; -} - -static inline unsigned int chcr_flits_to_desc(unsigned int n) -{ - WARN_ON(n > SGE_MAX_WR_LEN / 8); - return DIV_ROUND_UP(n, 8); -} -#endif /* __CHCR_COMMON_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b3570b41a737..40d51d2bd935 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -33,19 +33,8 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input); static void *chcr_uld_add(const struct cxgb4_lld_info *lld); static int chcr_uld_state_change(void *handle, enum cxgb4_state state); -#if defined(CONFIG_CHELSIO_TLS_DEVICE) -static const struct tlsdev_ops chcr_ktls_ops = { - .tls_dev_add = chcr_ktls_dev_add, - .tls_dev_del = chcr_ktls_dev_del, -}; -#endif - static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { [CPL_FW6_PLD] = cpl_fw6_pld_handler, -#ifdef CONFIG_CHELSIO_TLS_DEVICE - [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, - [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, -#endif }; static struct cxgb4_uld_info chcr_uld_info = { @@ -56,10 +45,6 @@ static struct cxgb4_uld_info chcr_uld_info = { .add = chcr_uld_add, .state_change = chcr_uld_state_change, .rx_handler = chcr_uld_rx_handler, -#if defined(CONFIG_CHELSIO_TLS_DEVICE) - .tx_handler = chcr_uld_tx_handler, - .tlsdev_ops = &chcr_ktls_ops, -#endif }; static void detach_work_fn(struct work_struct *work) @@ -235,15 +220,6 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } -#if defined(CONFIG_CHELSIO_TLS_DEVICE) -int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev) -{ - if (skb->decrypted) - return chcr_ktls_xmit(skb, dev); - return 0; -} -#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */ - static void chcr_detach_device(struct uld_ctx *u_ctx) { struct chcr_dev *dev = &u_ctx->dev; diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 81f6e61401e5..bb092b6b36b2 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -139,16 +139,4 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int err); int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); -#ifdef CONFIG_CHELSIO_TLS_DEVICE -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); -extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn); -extern void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction); -#endif #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c deleted file mode 100644 index c5cce024886a..000000000000 --- a/drivers/crypto/chelsio/chcr_ktls.c +++ /dev/null @@ -1,2017 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifdef CONFIG_CHELSIO_TLS_DEVICE -#include <linux/highmem.h> -#include "chcr_ktls.h" -#include "clip_tbl.h" - -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); -/* - * chcr_ktls_save_keys: calculate and save crypto keys. - * @tx_info - driver specific tls info. - * @crypto_info - tls crypto information. - * @direction - TX/RX direction. - * return - SUCCESS/FAILURE. - */ -static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, - struct tls_crypto_info *crypto_info, - enum tls_offload_ctx_dir direction) -{ - int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; - unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; - struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; - struct ktls_key_ctx *kctx = &tx_info->key_ctx; - struct crypto_cipher *cipher; - unsigned char *key, *salt; - - switch (crypto_info->cipher_type) { - case TLS_CIPHER_AES_GCM_128: - info_128_gcm = - (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; - mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; - tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; - tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); - - ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; - key = info_128_gcm->key; - salt = info_128_gcm->salt; - tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; - - /* The SCMD fields used when encrypting a full TLS - * record. Its a one time calculation till the - * connection exists. - */ - tx_info->scmd0_seqno_numivs = - SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | - SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | - SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | - SCMD_NUM_IVS_V(1); - - /* keys will be sent inline. */ - tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; - - /* The SCMD fields used when encrypting a partial TLS - * record (no trailer and possibly a truncated payload). - */ - tx_info->scmd0_short_seqno_numivs = - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | - SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); - - tx_info->scmd0_short_ivgen_hdrlen = - tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; - - break; - - default: - pr_err("GCM: cipher type 0x%x not supported\n", - crypto_info->cipher_type); - ret = -EINVAL; - goto out; - } - - key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + - roundup(keylen, 16) + ghash_size; - /* Calculate the H = CIPH(K, 0 repeated 16 times). - * It will go in key context - */ - cipher = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(cipher)) { - ret = -ENOMEM; - goto out; - } - - ret = crypto_cipher_setkey(cipher, key, keylen); - if (ret) - goto out1; - - memset(ghash_h, 0, ghash_size); - crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); - - /* fill the Key context */ - if (direction == TLS_OFFLOAD_CTX_DIR_TX) { - kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, - mac_key_size, - key_ctx_size >> 4); - } else { - ret = -EINVAL; - goto out1; - } - - memcpy(kctx->salt, salt, tx_info->salt_size); - memcpy(kctx->key, key, keylen); - memcpy(kctx->key + keylen, ghash_h, ghash_size); - tx_info->key_ctx_len = key_ctx_size; - -out1: - crypto_free_cipher(cipher); -out: - return ret; -} - -static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, - int new_state) -{ - /* This function can be called from both rx (interrupt context) and tx - * queue contexts. - */ - spin_lock_bh(&tx_info->lock); - switch (tx_info->connection_state) { - case KTLS_CONN_CLOSED: - tx_info->connection_state = new_state; - break; - - case KTLS_CONN_ACT_OPEN_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and also initialize TCB */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_ACT_OPEN_RPL: - /* if we are stuck in this state, means tcb init might not - * received by HW, try sending it again. - */ - if (!chcr_init_tcb_fields(tx_info)) - tx_info->connection_state = KTLS_CONN_SET_TCB_REQ; - break; - - case KTLS_CONN_SET_TCB_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and check if l2t_state is valid */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_SET_TCB_RPL: - /* Check if l2t state is valid, then move to ready state. */ - if (cxgb4_check_l2t_valid(tx_info->l2te)) { - tx_info->connection_state = KTLS_CONN_TX_READY; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx); - } - break; - - case KTLS_CONN_TX_READY: - /* nothing to be done here */ - break; - - default: - pr_err("unknown KTLS connection state\n"); - break; - } - spin_unlock_bh(&tx_info->lock); - - return tx_info->connection_state; -} -/* - * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req *cpl6; - struct cpl_act_open_req *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | - TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip = inet->inet_rcv_saddr; - cpl->peer_ip = inet->inet_daddr; - - /* fill first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} - -#if IS_ENABLED(CONFIG_IPV6) -/* - * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req6(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req6 *cpl6; - struct cpl_act_open_req6 *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req6 *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; - cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; - cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; - cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; - - /* first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} -#endif /* #if IS_ENABLED(CONFIG_IPV6) */ - -/* - * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_setup_connection(struct sock *sk, - struct chcr_ktls_info *tx_info) -{ - struct tid_info *t = &tx_info->adap->tids; - int atid, ret = 0; - - atid = cxgb4_alloc_atid(t, tx_info); - if (atid == -1) - return -EINVAL; - - tx_info->atid = atid; - tx_info->ip_family = sk->sk_family; - - if (sk->sk_family == AF_INET) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); - } else { - tx_info->ip_family = AF_INET6; - ret = cxgb4_clip_get(tx_info->netdev, - (const u32 *) - &sk->sk_v6_rcv_saddr.s6_addr, - 1); - if (ret) - goto out; - ret = chcr_ktls_act_open_req6(sk, tx_info, atid); - } -#endif - } - - /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret - * success, if any other return type clear atid and return that failure. - */ - if (ret) { - if (ret == NET_XMIT_CN) - ret = 0; - else - cxgb4_free_atid(t, atid); - goto out; - } - - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ); -out: - return ret; -} - -/* - * chcr_set_tcb_field: update tcb fields. - * @tx_info - driver specific tls info. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @no_reply - set 1 if not looking for TP response. - */ -static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, - u64 mask, u64 val, int no_reply) -{ - struct cpl_set_tcb_field *req; - struct sk_buff *skb; - - skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); - if (!skb) - return -ENOMEM; - - req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); - INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); - req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(no_reply)); - req->word_cookie = htons(TCB_WORD_V(word)); - req->mask = cpu_to_be64(mask); - req->val = cpu_to_be64(val); - - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - return cxgb4_ofld_send(tx_info->netdev, skb); -} - -/* - * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP. - */ -static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info) -{ - return chcr_set_tcb_field(tx_info, TCB_T_STATE_W, - TCB_T_STATE_V(TCB_T_STATE_M), - CHCR_TCB_STATE_CLOSED, 1); -} - -/* - * chcr_ktls_dev_del: call back for tls_dev_del. - * Remove the tid and l2t entry and close the connection. - * it per connection basis. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - */ -void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction) -{ - struct chcr_ktls_ofld_ctx_tx *tx_ctx = - chcr_get_ktls_tx_context(tls_ctx); - struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; - struct sock *sk; - - if (!tx_info) - return; - sk = tx_info->sk; - - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - /* clear l2t entry */ - if (tx_info->l2te) - cxgb4_l2t_release(tx_info->l2te); - -#if IS_ENABLED(CONFIG_IPV6) - /* clear clip entry */ - if (tx_info->ip_family == AF_INET6) - cxgb4_clip_release(netdev, - (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, - 1); -#endif - - /* clear tid */ - if (tx_info->tid != -1) { - /* clear tcb state and then release tid */ - chcr_ktls_mark_tcb_close(tx_info); - cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, - tx_info->tid, tx_info->ip_family); - } - - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close); - kvfree(tx_info); - tx_ctx->chcr_info = NULL; - /* release module refcount */ - module_put(THIS_MODULE); -} - -/* - * chcr_ktls_dev_add: call back for tls_dev_add. - * Create a tcb entry for TP. Also add l2t entry for the connection. And - * generate keys & save those keys locally. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - * return: SUCCESS/FAILURE. - */ -int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn) -{ - struct tls_context *tls_ctx = tls_get_ctx(sk); - struct chcr_ktls_ofld_ctx_tx *tx_ctx; - struct chcr_ktls_info *tx_info; - struct dst_entry *dst; - struct adapter *adap; - struct port_info *pi; - struct neighbour *n; - u8 daaddr[16]; - int ret = -1; - - tx_ctx = chcr_get_ktls_tx_context(tls_ctx); - - pi = netdev_priv(netdev); - adap = pi->adapter; - if (direction == TLS_OFFLOAD_CTX_DIR_RX) { - pr_err("not expecting for RX direction\n"); - ret = -EINVAL; - goto out; - } - if (tx_ctx->chcr_info) { - ret = -EINVAL; - goto out; - } - - tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); - if (!tx_info) { - ret = -ENOMEM; - goto out; - } - - spin_lock_init(&tx_info->lock); - - /* clear connection state */ - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - tx_info->sk = sk; - /* initialize tid and atid to -1, 0 is a also a valid id. */ - tx_info->tid = -1; - tx_info->atid = -1; - - tx_info->adap = adap; - tx_info->netdev = netdev; - tx_info->first_qset = pi->first_qset; - tx_info->tx_chan = pi->tx_chan; - tx_info->smt_idx = pi->smt_idx; - tx_info->port_id = pi->port_id; - - tx_info->rx_qid = chcr_get_first_rx_qid(adap); - if (unlikely(tx_info->rx_qid < 0)) - goto out2; - - tx_info->prev_seq = start_offload_tcp_sn; - tx_info->tcp_start_seq_number = start_offload_tcp_sn; - - /* save crypto keys */ - ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); - if (ret < 0) - goto out2; - - /* get peer ip */ - if (sk->sk_family == AF_INET) { - memcpy(daaddr, &sk->sk_daddr, 4); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) - memcpy(daaddr, &sk->sk_daddr, 4); - else - memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); -#endif - } - - /* get the l2t index */ - dst = sk_dst_get(sk); - if (!dst) { - pr_err("DST entry not found\n"); - goto out2; - } - n = dst_neigh_lookup(dst, daaddr); - if (!n || !n->dev) { - pr_err("neighbour not found\n"); - dst_release(dst); - goto out2; - } - tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); - - neigh_release(n); - dst_release(dst); - - if (!tx_info->l2te) { - pr_err("l2t entry not found\n"); - goto out2; - } - - tx_ctx->chcr_info = tx_info; - - /* create a filter and call cxgb4_l2t_send to send the packet out, which - * will take care of updating l2t entry in hw if not already done. - */ - ret = chcr_setup_connection(sk, tx_info); - if (ret) - goto out2; - - /* Driver shouldn't be removed until any single connection exists */ - if (!try_module_get(THIS_MODULE)) { - ret = -EINVAL; - goto out2; - } - - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open); - return 0; -out2: - kvfree(tx_info); -out: - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail); - return ret; -} - -/* - * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number - * handling. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) -{ - int ret = 0; - - /* set tcb in offload and bypass */ - ret = - chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, - TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), - TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); - if (ret) - return ret; - /* reset snd_una and snd_next fields in tcb */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, - TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | - TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), - 0, 1); - if (ret) - return ret; - - /* reset send max */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, - TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), - 0, 1); - if (ret) - return ret; - - /* update l2t index and request for tp reply to confirm tcb is - * initialised to handle tx traffic. - */ - ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, - TCB_L2T_IX_V(TCB_L2T_IX_M), - TCB_L2T_IX_V(tx_info->l2te->idx), 0); - return ret; -} - -/* - * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. - */ -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_act_open_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - unsigned int atid, tid, status; - struct tid_info *t; - - tid = GET_TID(p); - status = AOPEN_STATUS_G(ntohl(p->atid_status)); - atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); - - t = &adap->tids; - tx_info = lookup_atid(t, atid); - - if (!tx_info || tx_info->atid != atid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - - if (!status) { - tx_info->tid = tid; - cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); - - cxgb4_free_atid(t, atid); - tx_info->atid = -1; - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, - KTLS_CONN_ACT_OPEN_RPL); - } - return 0; -} - -/* - * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. - */ -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_set_tcb_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - struct tid_info *t; - u32 tid; - - tid = GET_TID(p); - - t = &adap->tids; - tx_info = lookup_tid(t, tid); - if (!tx_info || tx_info->tid != tid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL); - return 0; -} - -static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - u32 tid, void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - struct cpl_set_tcb_field_core *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *txpkt; - - /* ULP_TXPKT */ - txpkt = pos; - txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); - txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); - - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(txpkt + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); - idata->len = htonl(sizeof(*cpl)); - pos = idata + 1; - - cpl = pos; - /* CPL_SET_TCB_FIELD */ - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); - cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(!reply)); - cpl->word_cookie = htons(TCB_WORD_V(word)); - cpl->mask = cpu_to_be64(mask); - cpl->val = cpu_to_be64(val); - - /* ULPTX_NOOP */ - idata = (struct ulptx_idata *)(cpl + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); - idata->len = htonl(0); - pos = idata + 1; - - return pos; -} - - -/* - * chcr_write_cpl_set_tcb_ulp: update tcb values. - * TCB is responsible to create tcp headers, so all the related values - * should be correctly updated. - * @tx_info - driver specific tls info. - * @q - tx queue on which packet is going out. - * @tid - TCB identifier. - * @pos - current index where should we start writing. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @reply - set 1 if looking for TP response. - * return - next position to write. - */ -static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tid, - void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - int left = (void *)q->q.stat - pos; - - if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { - if (!left) { - pos = q->q.desc; - } else { - u8 buf[48] = {0}; - - __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, - mask, val, reply); - - return chcr_copy_to_txd(buf, &q->q, pos, - CHCR_SET_TCB_FIELD_LEN); - } - } - - pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, - mask, val, reply); - - /* check again if we are at the end of the queue */ - if (left == CHCR_SET_TCB_FIELD_LEN) - pos = q->q.desc; - - return pos; -} - -/* - * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header - * with updated values like tcp seq, ack, window etc. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_ack - * @tcp_win - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u64 tcp_seq, - u64 tcp_ack, u64 tcp_win) -{ - bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); - u32 len, cpl = 0, ndesc, wr_len; - struct fw_ulptx_wr *wr; - int credits; - void *pos; - - wr_len = sizeof(*wr); - /* there can be max 4 cpls, check if we have enough credits */ - len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); - ndesc = DIV_ROUND_UP(len, 64); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - /* make space for WR, we'll fill it later when we know all the cpls - * being sent out and have complete length. - */ - wr = pos; - pos += wr_len; - /* update tx_max if its a re-transmit or the first wr */ - if (first_wr || tcp_seq != tx_info->prev_seq) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_TX_MAX_W, - TCB_TX_MAX_V(TCB_TX_MAX_M), - TCB_TX_MAX_V(tcp_seq), 0); - cpl++; - } - /* reset snd una if it's a re-transmit pkt */ - if (tcp_seq != tx_info->prev_seq) { - /* reset snd_una */ - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_SND_UNA_RAW_W, - TCB_SND_UNA_RAW_V - (TCB_SND_UNA_RAW_M), - TCB_SND_UNA_RAW_V(0), 0); - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo); - cpl++; - } - /* update ack */ - if (first_wr || tx_info->prev_ack != tcp_ack) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_NXT_W, - TCB_RCV_NXT_V(TCB_RCV_NXT_M), - TCB_RCV_NXT_V(tcp_ack), 0); - tx_info->prev_ack = tcp_ack; - cpl++; - } - /* update receive window */ - if (first_wr || tx_info->prev_win != tcp_win) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_WND_W, - TCB_RCV_WND_V(TCB_RCV_WND_M), - TCB_RCV_WND_V(tcp_wi |